Merge pull request #484 from TeamNewPipe/yt_,music_search

Fix YouTube Music search

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java

@@ -74,7 +74,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
      * however it should not be overridden by the actual implementation.
      *
      * @param id
-     * @return the url coresponding to id without any filters applied
+     * @return the url corresponding to id without any filters applied
      */
     public String getUrl(String id) throws ParsingException {
         return getUrl(id, new ArrayList<String>(0), "");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -205,12 +205,12 @@ public class YoutubeParsingHelper {
 
     /**
      * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
-     * Ids from a YouTube Music Mix start with "RDAMVM"
+     * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
      * @param playlistId
      * @return Whether given id belongs to a YouTube Music Mix
      */
     public static boolean isYoutubeMusicMixId(final String playlistId) {
-        return playlistId.startsWith("RDAMVM");
+        return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
     }
     /**
      * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
@@ -226,20 +226,20 @@ public class YoutubeParsingHelper {
      * @throws ParsingException If the playlistId is a Channel Mix or not a mix.
      */
     public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
-        if (playlistId.startsWith("RDMM")) { //My Mix
+        if (playlistId.startsWith("RDMM")) { // My Mix
             return playlistId.substring(4);
 
-        } else if (playlistId.startsWith("RDAMVM")) { //Music mix
+        } else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK"
             return playlistId.substring(6);
 
-        } else if (playlistId.startsWith("RMCM")) { //Channel mix
+        } else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
-            //Channel mix are build with RMCM{channelId}, so videoId can't be determined
+            // Channel mix are build with RMCM{channelId}, so videoId can't be determined
             throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
 
-        } else if (playlistId.startsWith("RD")) { // Normal mix
+        } else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
             return playlistId.substring(2);
 
-        } else { //not a mix
+        } else { // not a mix
             throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -111,7 +111,8 @@ public class YoutubeService extends StreamingService {
 
     @Override
     public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
-        if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
+        if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())
+                && !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) {
             return new YoutubeMixPlaylistExtractor(this, linkHandler);
         } else {
             return new YoutubePlaylistExtractor(this, linkHandler);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -20,6 +20,8 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -258,16 +260,29 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
         for (Object item : videos) {
-            final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null);
+            final JsonObject info = ((JsonObject) item)
+                    .getObject("musicResponsiveListItemRenderer", null);
             if (info != null) {
+                final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING);
+                if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) {
+                    continue; // no info about video URL available
+                }
+
+                final JsonObject flexColumnRenderer = info
+                        .getArray("flexColumns")
+                        .getObject(1)
+                        .getObject("musicResponsiveListItemFlexColumnRenderer");
+                final JsonArray descriptionElements = flexColumnRenderer
+                        .getObject("text")
+                        .getArray("runs");
                 final String searchType = getLinkHandler().getContentFilters().get(0);
                 if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
                     collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
                         @Override
                         public String getUrl() throws ParsingException {
-                            final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
+                            final String id = info.getObject("playlistItemData").getString("videoId");
-                            if (!isNullOrEmpty(url)) {
+                            if (!isNullOrEmpty(id)) {
-                                return url;
+                                return "https://music.youtube.com/watch?v=" + id;
                             }
                             throw new ParsingException("Could not get url");
                         }
@@ -284,8 +299,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
 
                         @Override
                         public long getDuration() throws ParsingException {
-                            final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
+                            final String duration = descriptionElements
-                                    .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
+                                    .getObject(descriptionElements.size() - 1)
+                                    .getString("text");
                             if (!isNullOrEmpty(duration)) {
                                 return YoutubeParsingHelper.parseDurationString(duration);
                             }
@@ -294,8 +310,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
 
                         @Override
                         public String getUploaderName() throws ParsingException {
-                            final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
+                            final String name = descriptionElements.getObject(0).getString("text");
-                                    .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
                             if (!isNullOrEmpty(name)) {
                                 return name;
                             }
@@ -346,8 +361,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                             if (searchType.equals(MUSIC_SONGS)) {
                                 return -1;
                             }
-                            final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
+                            final String viewCount = descriptionElements
-                                    .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
+                                    .getObject(descriptionElements.size() - 3)
+                                    .getString("text");
                             if (!isNullOrEmpty(viewCount)) {
                                 return Utils.mixedNumberWordToLong(viewCount);
                             }
@@ -451,9 +467,27 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
 
                         @Override
                         public String getUrl() throws ParsingException {
-                            final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
+                            String playlistId = info.getObject("menu")
-                            if (!isNullOrEmpty(url)) {
+                                    .getObject("menuRenderer")
-                                return url;
+                                    .getArray("items")
+                                    .getObject(4)
+                                    .getObject("toggleMenuServiceItemRenderer")
+                                    .getObject("toggledServiceEndpoint")
+                                    .getObject("likeEndpoint")
+                                    .getObject("target")
+                                    .getString("playlistId");
+
+                            if (isNullOrEmpty(playlistId)) {
+                                playlistId = info.getObject("overlay")
+                                        .getObject("musicItemThumbnailOverlayRenderer")
+                                        .getObject("content")
+                                        .getObject("musicPlayButtonRenderer")
+                                        .getObject("playNavigationEndpoint")
+                                        .getObject("watchPlaylistEndpoint")
+                                        .getString("playlistId");
+                            }
+                            if (!isNullOrEmpty(playlistId)) {
+                                return "https://music.youtube.com/playlist?list=" + playlistId;
                             }
                             throw new ParsingException("Could not get url");
                         }
@@ -462,11 +496,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                         public String getUploaderName() throws ParsingException {
                             final String name;
                             if (searchType.equals(MUSIC_ALBUMS)) {
-                                name = getTextFromObject(info.getArray("flexColumns").getObject(2)
+                                name = descriptionElements.getObject(2).getString("text");
-                                        .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
                             } else {
-                                name = getTextFromObject(info.getArray("flexColumns").getObject(1)
+                                name = descriptionElements.getObject(0).getString("text");
-                                        .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
                             }
                             if (!isNullOrEmpty(name)) {
                                 return name;
@@ -479,8 +511,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                             if (searchType.equals(MUSIC_ALBUMS)) {
                                 return ITEM_COUNT_UNKNOWN;
                             }
-                            final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
+                            final String count = descriptionElements.getObject(2).getString("text");
-                                    .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
                             if (!isNullOrEmpty(count)) {
                                 if (count.contains("100+")) {
                                     return ITEM_COUNT_MORE_THAN_100;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java

@@ -52,11 +52,6 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
                         "the list-ID given in the URL does not match the list pattern");
             }
 
-            if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) {
-                throw new ContentNotSupportedException(
-                        "YouTube Music Mix playlists are not yet supported");
-            }
-
             if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
                     && Utils.getQueryValue(urlObj, "v") == null) {
                 //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId