Merge pull request #1240 from AudricV/yt_fix-playlists-items-extraction

[YouTube] Add support for new playlist items data structure

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java

@@ -18,6 +18,7 @@ import java.util.Optional;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -363,27 +364,11 @@ public final class YoutubeChannelHelper {
                 final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT)
                         .getObject(PAGE_HEADER_VIEW_MODEL);
 
-                final boolean hasCircleOrMusicIcon = pageHeaderViewModel.getObject(TITLE)
+                final boolean hasCircleOrMusicIcon = hasArtistOrVerifiedIconBadgeAttachment(
-                        .getObject("dynamicTextViewModel")
+                        pageHeaderViewModel.getObject(TITLE)
-                        .getObject("text")
+                                .getObject("dynamicTextViewModel")
-                        .getArray("attachmentRuns")
+                                .getObject("text")
-                        .stream()
+                                .getArray("attachmentRuns"));
-                        .filter(JsonObject.class::isInstance)
-                        .map(JsonObject.class::cast)
-                        .anyMatch(attachmentRun -> attachmentRun.getObject("element")
-                                .getObject("type")
-                                .getObject("imageType")
-                                .getObject("image")
-                                .getArray("sources")
-                                .stream()
-                                .filter(JsonObject.class::isInstance)
-                                .map(JsonObject.class::cast)
-                                .anyMatch(source -> {
-                                    final String imageName = source.getObject("clientResource")
-                                            .getString("imageName");
-                                    return "CHECK_CIRCLE_FILLED".equals(imageName)
-                                            || "MUSIC_FILLED".equals(imageName);
-                                }));
                 if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image")
                         .has("contentPreviewImageViewModel")) {
                     // If a pageHeaderRenderer has no object in which a check verified may be

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1584,6 +1584,29 @@ public final class YoutubeParsingHelper {
         return false;
     }
 
+    public static boolean hasArtistOrVerifiedIconBadgeAttachment(
+            @Nonnull final JsonArray attachmentRuns) {
+        return attachmentRuns.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .anyMatch(attachmentRun -> attachmentRun.getObject("element")
+                        .getObject("type")
+                        .getObject("imageType")
+                        .getObject("image")
+                        .getArray("sources")
+                        .stream()
+                        .filter(JsonObject.class::isInstance)
+                        .map(JsonObject.class::cast)
+                        .anyMatch(source -> {
+                            final String imageName = source.getObject("clientResource")
+                                    .getString("imageName");
+                            return "CHECK_CIRCLE_FILLED".equals(imageName)
+                                    || "AUDIO_BADGE".equals(imageName)
+                                    || "MUSIC_FILLED".equals(imageName);
+                        }));
+
+    }
+
     /**
      * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
      * playback requests (and also for some clients, in the player request body).

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java

@@ -311,6 +311,12 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
         } else if (item.has("expandedShelfContentsRenderer")) {
             return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
                     .getArray("items"), channelVerifiedStatus, channelName, channelUrl);
+        } else if (item.has("lockupViewModel")) {
+            final JsonObject lockupViewModel = item.getObject("lockupViewModel");
+            if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) {
+                commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus,
+                        channelName, channelUrl);
+            }
         } else if (item.has("continuationItemRenderer")) {
             return Optional.ofNullable(item.getObject("continuationItemRenderer"));
         }
@@ -366,6 +372,37 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
                 });
     }
 
+    private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector,
+                                      @Nonnull final JsonObject playlistLockupViewModel,
+                                      @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                      @Nullable final String channelName,
+                                      @Nullable final String channelUrl) {
+        collector.commit(
+                new YoutubeMixOrPlaylistLockupInfoItemExtractor(playlistLockupViewModel) {
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @Override
+                    public boolean isUploaderVerified() throws ParsingException {
+                        switch (channelVerifiedStatus) {
+                            case VERIFIED:
+                                return true;
+                            case UNVERIFIED:
+                                return false;
+                            default:
+                                return super.isUploaderVerified();
+                        }
+                    }
+                });
+    }
+
     private void commitVideo(@Nonnull final MultiInfoItemsCollector collector,
                              @Nonnull final TimeAgoParser timeAgoParser,
                              @Nonnull final JsonObject jsonObject,

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java

@@ -0,0 +1,193 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
+
+public class YoutubeMixOrPlaylistLockupInfoItemExtractor implements PlaylistInfoItemExtractor {
+
+    @Nonnull
+    private final JsonObject lockupViewModel;
+    @Nonnull
+    private final JsonObject thumbnailViewModel;
+    @Nonnull
+    private final JsonObject lockupMetadataViewModel;
+    @Nonnull
+    private final JsonObject firstMetadataRow;
+    @Nonnull
+    private PlaylistInfo.PlaylistType playlistType;
+
+    public YoutubeMixOrPlaylistLockupInfoItemExtractor(@Nonnull final JsonObject lockupViewModel) {
+        this.lockupViewModel = lockupViewModel;
+        this.thumbnailViewModel = lockupViewModel.getObject("contentImage")
+                .getObject("collectionThumbnailViewModel")
+                .getObject("primaryThumbnail")
+                .getObject("thumbnailViewModel");
+        this.lockupMetadataViewModel = lockupViewModel.getObject("metadata")
+                .getObject("lockupMetadataViewModel");
+        /*
+        The metadata rows are structured in the following way:
+        1st part: uploader info, playlist type, playlist updated date
+        2nd part: space row
+        3rd element: first video
+        4th (not always returned for playlists with less than 2 items?): second video
+        5th element (always returned, but at a different index for playlists with less than 2
+        items?): Show full playlist
+
+        The first metadata row has the following structure:
+        1st array element: uploader info
+        2nd element: playlist type (course, playlist, podcast)
+        3rd element (not always returned): playlist updated date
+         */
+        this.firstMetadataRow = lockupMetadataViewModel.getObject("metadata")
+                .getObject("contentMetadataViewModel")
+                .getArray("metadataRows")
+                .getObject(0);
+
+        try {
+            this.playlistType = extractPlaylistTypeFromPlaylistId(getPlaylistId());
+        } catch (final ParsingException e) {
+            // If we cannot extract the playlist type, fall back to the normal one
+            this.playlistType = PlaylistInfo.PlaylistType.NORMAL;
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return firstMetadataRow.getArray("metadataParts")
+                .getObject(0)
+                .getObject("text")
+                .getString("content");
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, there is no uploader as they are auto-generated
+            return null;
+        }
+
+        return getUrlFromNavigationEndpoint(
+                firstMetadataRow.getArray("metadataParts")
+                        .getObject(0)
+                        .getObject("text")
+                        .getArray("commandRuns")
+                        .getObject(0)
+                        .getObject("onTap")
+                        .getObject("innertubeCommand"));
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, there is no uploader as they are auto-generated
+            return false;
+        }
+
+        return hasArtistOrVerifiedIconBadgeAttachment(
+                firstMetadataRow.getArray("metadataParts")
+                        .getObject(0)
+                        .getObject("text")
+                        .getArray("attachmentRuns"));
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, we are not able to get its stream count
+            return ListExtractor.ITEM_COUNT_INFINITE;
+        }
+
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(
+                    thumbnailViewModel.getArray("overlays")
+                            .stream()
+                            .filter(JsonObject.class::isInstance)
+                            .map(JsonObject.class::cast)
+                            .filter(overlay -> overlay.has("thumbnailOverlayBadgeViewModel"))
+                            .findFirst()
+                            .orElseThrow(() -> new ParsingException(
+                                    "Could not get thumbnailOverlayBadgeViewModel"))
+                            .getObject("thumbnailOverlayBadgeViewModel")
+                            .getArray("thumbnailBadges")
+                            .stream()
+                            .filter(JsonObject.class::isInstance)
+                            .map(JsonObject.class::cast)
+                            .filter(badge -> badge.has("thumbnailBadgeViewModel"))
+                            .findFirst()
+                            .orElseThrow(() ->
+                                    new ParsingException("Could not get thumbnailBadgeViewModel"))
+                            .getObject("thumbnailBadgeViewModel")
+                            .getString("text")));
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get playlist stream count", e);
+        }
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return lockupMetadataViewModel.getObject("title")
+                .getString("content");
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        // If the playlist item is a mix, we cannot return just its playlist ID as mix playlists
+        // are not viewable in playlist pages
+        // Use directly getUrlFromNavigationEndpoint in this case, which returns the watch URL with
+        // the mix playlist
+        if (playlistType == PlaylistInfo.PlaylistType.NORMAL) {
+            try {
+                return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(getPlaylistId());
+            } catch (final Exception ignored) {
+            }
+        }
+
+        return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext")
+                .getObject("commandContext")
+                .getObject("onTap")
+                .getObject("innertubeCommand"));
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getThumbnails() throws ParsingException {
+        return getImagesFromThumbnailsArray(thumbnailViewModel.getObject("image")
+                .getArray("sources"));
+    }
+
+    @Nonnull
+    @Override
+    public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
+        return playlistType;
+    }
+
+    private String getPlaylistId() throws ParsingException {
+        String id = lockupViewModel.getString("contentId");
+        if (Utils.isNullOrEmpty(id)) {
+            id = lockupViewModel.getObject("rendererContext")
+                    .getObject("commandContext")
+                    .getObject("watchEndpoint")
+                    .getString("playlistId");
+        }
+
+        if (Utils.isNullOrEmpty(id)) {
+            throw new ParsingException("Could not get playlist ID");
+        }
+
+        return id;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -245,6 +245,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
                 } else if (item.has("showRenderer")) {
                     collector.commit(new YoutubeShowRendererInfoItemExtractor(
                             item.getObject("showRenderer")));
+                } else if (item.has("lockupViewModel")) {
+                    final JsonObject lockupViewModel = item.getObject("lockupViewModel");
+                    if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
+                            lockupViewModel.getString("contentType"))) {
+                        collector.commit(
+                                new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel));
+                    }
                 }
             }
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -726,6 +726,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                         } else if (result.has("compactPlaylistRenderer")) {
                             return new YoutubeMixOrPlaylistInfoItemExtractor(
                                     result.getObject("compactPlaylistRenderer"));
+                        } else if (result.has("lockupViewModel")) {
+                            final JsonObject lockupViewModel = result.getObject("lockupViewModel");
+                            if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
+                                    lockupViewModel.getString("contentType"))) {
+                                return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
+                                        lockupViewModel);
+                            }
                         }
                         return null;
                     })