From c9d9bd1e249020f2504761bee64ca3826ad6a720 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Thu, 17 Dec 2020 10:18:07 +0100 Subject: [PATCH 1/4] Fix typo in ListLinkHandlerFactory#getUrl --- .../newpipe/extractor/linkhandler/ListLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 4980c3191..86991c4df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -74,7 +74,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { * however it should not be overridden by the actual implementation. * * @param id - * @return the url coresponding to id without any filters applied + * @return the url corresponding to id without any filters applied */ public String getUrl(String id) throws ParsingException { return getUrl(id, new ArrayList(0), ""); From b62144b49d51864912cb4542e216458e27b55965 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 22 Dec 2020 20:02:36 +0100 Subject: [PATCH 2/4] [YouTube] Music: Fix search extraction caused by changed JSON response --- .../YoutubeMusicSearchExtractor.java | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index c7a4af8ce..1c583b9be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -20,6 +20,8 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -258,16 +260,29 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { - final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null); + final JsonObject info = ((JsonObject) item) + .getObject("musicResponsiveListItemRenderer", null); if (info != null) { + final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING); + if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { + continue; // no info about video URL available + } + + final JsonObject flexColumnRenderer = info + .getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer"); + final JsonArray descriptionElements = flexColumnRenderer + .getObject("text") + .getArray("runs"); final String searchType = getLinkHandler().getContentFilters().get(0); if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (!isNullOrEmpty(url)) { - return url; + final String id = info.getObject("playlistItemData").getString("videoId"); + if (!isNullOrEmpty(id)) { + return "https://music.youtube.com/watch?v=" + id; } throw new ParsingException("Could not get url"); } @@ -284,8 +299,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public long getDuration() throws ParsingException { - final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String duration = descriptionElements + .getObject(descriptionElements.size() - 1) + .getString("text"); if (!isNullOrEmpty(duration)) { return YoutubeParsingHelper.parseDurationString(duration); } @@ -294,8 +310,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUploaderName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String name = descriptionElements.getObject(0).getString("text"); if (!isNullOrEmpty(name)) { return name; } @@ -346,8 +361,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { if (searchType.equals(MUSIC_SONGS)) { return -1; } - final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String viewCount = descriptionElements + .getObject(descriptionElements.size() - 3) + .getString("text"); if (!isNullOrEmpty(viewCount)) { return Utils.mixedNumberWordToLong(viewCount); } @@ -451,9 +467,27 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (!isNullOrEmpty(url)) { - return url; + String playlistId = info.getObject("menu") + .getObject("menuRenderer") + .getArray("items") + .getObject(4) + .getObject("toggleMenuServiceItemRenderer") + .getObject("toggledServiceEndpoint") + .getObject("likeEndpoint") + .getObject("target") + .getString("playlistId"); + + if (isNullOrEmpty(playlistId)) { + playlistId = info.getObject("overlay") + .getObject("musicItemThumbnailOverlayRenderer") + .getObject("content") + .getObject("musicPlayButtonRenderer") + .getObject("playNavigationEndpoint") + .getObject("watchPlaylistEndpoint") + .getString("playlistId"); + } + if (!isNullOrEmpty(playlistId)) { + return "https://music.youtube.com/playlist?list=" + playlistId; } throw new ParsingException("Could not get url"); } @@ -462,11 +496,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getUploaderName() throws ParsingException { final String name; if (searchType.equals(MUSIC_ALBUMS)) { - name = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + name = descriptionElements.getObject(2).getString("text"); } else { - name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + name = descriptionElements.getObject(0).getString("text"); } if (!isNullOrEmpty(name)) { return name; @@ -479,8 +511,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { if (searchType.equals(MUSIC_ALBUMS)) { return ITEM_COUNT_UNKNOWN; } - final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String count = descriptionElements.getObject(2).getString("text"); if (!isNullOrEmpty(count)) { if (count.contains("100+")) { return ITEM_COUNT_MORE_THAN_100; From 3033c0b99334489a64c78e042bde3443073f4341 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 23 Dec 2020 21:07:30 +0100 Subject: [PATCH 3/4] [YouTube] Music Mix: Fix playlist not being accepted Regression introduced by YouTube Mix support (#280) --- .../extractor/services/youtube/YoutubeParsingHelper.java | 6 +++--- .../newpipe/extractor/services/youtube/YoutubeService.java | 3 ++- .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 5 ----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 05749bbdb..48589cc33 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -205,12 +205,12 @@ public class YoutubeParsingHelper { /** * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) - * Ids from a YouTube Music Mix start with "RDAMVM" + * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" * @param playlistId * @return Whether given id belongs to a YouTube Music Mix */ public static boolean isYoutubeMusicMixId(final String playlistId) { - return playlistId.startsWith("RDAMVM"); + return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK"); } /** * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) @@ -229,7 +229,7 @@ public class YoutubeParsingHelper { if (playlistId.startsWith("RDMM")) { //My Mix return playlistId.substring(4); - } else if (playlistId.startsWith("RDAMVM")) { //Music mix + } else if (playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK")) { //Music mix return playlistId.substring(6); } else if (playlistId.startsWith("RMCM")) { //Channel mix diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 997fd0b73..6e7ab7407 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -111,7 +111,8 @@ public class YoutubeService extends StreamingService { @Override public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { - if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId()) + && !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) { return new YoutubeMixPlaylistExtractor(this, linkHandler); } else { return new YoutubePlaylistExtractor(this, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index aa2908e64..74b8f91f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -52,11 +52,6 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { "the list-ID given in the URL does not match the list pattern"); } - if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { - throw new ContentNotSupportedException( - "YouTube Music Mix playlists are not yet supported"); - } - if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) && Utils.getQueryValue(urlObj, "v") == null) { //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId From d9e2da53c3b781c77f7726da6131f3e01c28e161 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 25 Dec 2020 15:00:31 +0100 Subject: [PATCH 4/4] =?UTF-8?q?Replace=20explicit=20string=20checks=20whet?= =?UTF-8?q?her=20a=20playlist=20=C3=ADd=20is=20a=20certain=20YouTube=20Mix?= =?UTF-8?q?=20type=20with=20calling=20the=20dedicated=20methods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/youtube/YoutubeParsingHelper.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 48589cc33..40d15288a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -226,20 +226,20 @@ public class YoutubeParsingHelper { * @throws ParsingException If the playlistId is a Channel Mix or not a mix. */ public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { - if (playlistId.startsWith("RDMM")) { //My Mix + if (playlistId.startsWith("RDMM")) { // My Mix return playlistId.substring(4); - } else if (playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK")) { //Music mix + } else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK" return playlistId.substring(6); - } else if (playlistId.startsWith("RMCM")) { //Channel mix - //Channel mix are build with RMCM{channelId}, so videoId can't be determined + } else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM" + // Channel mix are build with RMCM{channelId}, so videoId can't be determined throw new ParsingException("Video id could not be determined from mix id: " + playlistId); - } else if (playlistId.startsWith("RD")) { // Normal mix + } else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD" return playlistId.substring(2); - } else { //not a mix + } else { // not a mix throw new ParsingException("Video id could not be determined from mix id: " + playlistId); } }