diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index 54d29032e..5eb852b0b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -18,6 +18,7 @@ import java.util.Optional; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -363,27 +364,11 @@ public final class YoutubeChannelHelper { final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT) .getObject(PAGE_HEADER_VIEW_MODEL); - final boolean hasCircleOrMusicIcon = pageHeaderViewModel.getObject(TITLE) - .getObject("dynamicTextViewModel") - .getObject("text") - .getArray("attachmentRuns") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .anyMatch(attachmentRun -> attachmentRun.getObject("element") - .getObject("type") - .getObject("imageType") - .getObject("image") - .getArray("sources") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .anyMatch(source -> { - final String imageName = source.getObject("clientResource") - .getString("imageName"); - return "CHECK_CIRCLE_FILLED".equals(imageName) - || "MUSIC_FILLED".equals(imageName); - })); + final boolean hasCircleOrMusicIcon = hasArtistOrVerifiedIconBadgeAttachment( + pageHeaderViewModel.getObject(TITLE) + .getObject("dynamicTextViewModel") + .getObject("text") + .getArray("attachmentRuns")); if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image") .has("contentPreviewImageViewModel")) { // If a pageHeaderRenderer has no object in which a check verified may be diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index c0aee17cc..f514b61d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1584,6 +1584,29 @@ public final class YoutubeParsingHelper { return false; } + public static boolean hasArtistOrVerifiedIconBadgeAttachment( + @Nonnull final JsonArray attachmentRuns) { + return attachmentRuns.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .anyMatch(attachmentRun -> attachmentRun.getObject("element") + .getObject("type") + .getObject("imageType") + .getObject("image") + .getArray("sources") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .anyMatch(source -> { + final String imageName = source.getObject("clientResource") + .getString("imageName"); + return "CHECK_CIRCLE_FILLED".equals(imageName) + || "AUDIO_BADGE".equals(imageName) + || "MUSIC_FILLED".equals(imageName); + })); + + } + /** * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in * playback requests (and also for some clients, in the player request body). diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 9bd4b3398..9d2a9e947 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -311,6 +311,12 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { } else if (item.has("expandedShelfContentsRenderer")) { return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer") .getArray("items"), channelVerifiedStatus, channelName, channelUrl); + } else if (item.has("lockupViewModel")) { + final JsonObject lockupViewModel = item.getObject("lockupViewModel"); + if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) { + commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus, + channelName, channelUrl); + } } else if (item.has("continuationItemRenderer")) { return Optional.ofNullable(item.getObject("continuationItemRenderer")); } @@ -366,6 +372,37 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { }); } + private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector, + @Nonnull final JsonObject playlistLockupViewModel, + @Nonnull final VerifiedStatus channelVerifiedStatus, + @Nullable final String channelName, + @Nullable final String channelUrl) { + collector.commit( + new YoutubeMixOrPlaylistLockupInfoItemExtractor(playlistLockupViewModel) { + @Override + public String getUploaderName() throws ParsingException { + return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName; + } + + @Override + public String getUploaderUrl() throws ParsingException { + return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl; + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + switch (channelVerifiedStatus) { + case VERIFIED: + return true; + case UNVERIFIED: + return false; + default: + return super.isUploaderVerified(); + } + } + }); + } + private void commitVideo(@Nonnull final MultiInfoItemsCollector collector, @Nonnull final TimeAgoParser timeAgoParser, @Nonnull final JsonObject jsonObject, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java new file mode 100644 index 000000000..009fc51d0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java @@ -0,0 +1,193 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfo; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment; + +public class YoutubeMixOrPlaylistLockupInfoItemExtractor implements PlaylistInfoItemExtractor { + + @Nonnull + private final JsonObject lockupViewModel; + @Nonnull + private final JsonObject thumbnailViewModel; + @Nonnull + private final JsonObject lockupMetadataViewModel; + @Nonnull + private final JsonObject firstMetadataRow; + @Nonnull + private PlaylistInfo.PlaylistType playlistType; + + public YoutubeMixOrPlaylistLockupInfoItemExtractor(@Nonnull final JsonObject lockupViewModel) { + this.lockupViewModel = lockupViewModel; + this.thumbnailViewModel = lockupViewModel.getObject("contentImage") + .getObject("collectionThumbnailViewModel") + .getObject("primaryThumbnail") + .getObject("thumbnailViewModel"); + this.lockupMetadataViewModel = lockupViewModel.getObject("metadata") + .getObject("lockupMetadataViewModel"); + /* + The metadata rows are structured in the following way: + 1st part: uploader info, playlist type, playlist updated date + 2nd part: space row + 3rd element: first video + 4th (not always returned for playlists with less than 2 items?): second video + 5th element (always returned, but at a different index for playlists with less than 2 + items?): Show full playlist + + The first metadata row has the following structure: + 1st array element: uploader info + 2nd element: playlist type (course, playlist, podcast) + 3rd element (not always returned): playlist updated date + */ + this.firstMetadataRow = lockupMetadataViewModel.getObject("metadata") + .getObject("contentMetadataViewModel") + .getArray("metadataRows") + .getObject(0); + + try { + this.playlistType = extractPlaylistTypeFromPlaylistId(getPlaylistId()); + } catch (final ParsingException e) { + // If we cannot extract the playlist type, fall back to the normal one + this.playlistType = PlaylistInfo.PlaylistType.NORMAL; + } + } + + @Override + public String getUploaderName() throws ParsingException { + return firstMetadataRow.getArray("metadataParts") + .getObject(0) + .getObject("text") + .getString("content"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (playlistType != PlaylistInfo.PlaylistType.NORMAL) { + // If the playlist is a mix, there is no uploader as they are auto-generated + return null; + } + + return getUrlFromNavigationEndpoint( + firstMetadataRow.getArray("metadataParts") + .getObject(0) + .getObject("text") + .getArray("commandRuns") + .getObject(0) + .getObject("onTap") + .getObject("innertubeCommand")); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + if (playlistType != PlaylistInfo.PlaylistType.NORMAL) { + // If the playlist is a mix, there is no uploader as they are auto-generated + return false; + } + + return hasArtistOrVerifiedIconBadgeAttachment( + firstMetadataRow.getArray("metadataParts") + .getObject(0) + .getObject("text") + .getArray("attachmentRuns")); + } + + @Override + public long getStreamCount() throws ParsingException { + if (playlistType != PlaylistInfo.PlaylistType.NORMAL) { + // If the playlist is a mix, we are not able to get its stream count + return ListExtractor.ITEM_COUNT_INFINITE; + } + + try { + return Long.parseLong(Utils.removeNonDigitCharacters( + thumbnailViewModel.getArray("overlays") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(overlay -> overlay.has("thumbnailOverlayBadgeViewModel")) + .findFirst() + .orElseThrow(() -> new ParsingException( + "Could not get thumbnailOverlayBadgeViewModel")) + .getObject("thumbnailOverlayBadgeViewModel") + .getArray("thumbnailBadges") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(badge -> badge.has("thumbnailBadgeViewModel")) + .findFirst() + .orElseThrow(() -> + new ParsingException("Could not get thumbnailBadgeViewModel")) + .getObject("thumbnailBadgeViewModel") + .getString("text"))); + } catch (final Exception e) { + throw new ParsingException("Could not get playlist stream count", e); + } + } + + @Override + public String getName() throws ParsingException { + return lockupMetadataViewModel.getObject("title") + .getString("content"); + } + + @Override + public String getUrl() throws ParsingException { + // If the playlist item is a mix, we cannot return just its playlist ID as mix playlists + // are not viewable in playlist pages + // Use directly getUrlFromNavigationEndpoint in this case, which returns the watch URL with + // the mix playlist + if (playlistType == PlaylistInfo.PlaylistType.NORMAL) { + try { + return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(getPlaylistId()); + } catch (final Exception ignored) { + } + } + + return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext") + .getObject("commandContext") + .getObject("onTap") + .getObject("innertubeCommand")); + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + return getImagesFromThumbnailsArray(thumbnailViewModel.getObject("image") + .getArray("sources")); + } + + @Nonnull + @Override + public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { + return playlistType; + } + + private String getPlaylistId() throws ParsingException { + String id = lockupViewModel.getString("contentId"); + if (Utils.isNullOrEmpty(id)) { + id = lockupViewModel.getObject("rendererContext") + .getObject("commandContext") + .getObject("watchEndpoint") + .getString("playlistId"); + } + + if (Utils.isNullOrEmpty(id)) { + throw new ParsingException("Could not get playlist ID"); + } + + return id; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index a0be72cc8..f10db0bc8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -245,6 +245,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { } else if (item.has("showRenderer")) { collector.commit(new YoutubeShowRendererInfoItemExtractor( item.getObject("showRenderer"))); + } else if (item.has("lockupViewModel")) { + final JsonObject lockupViewModel = item.getObject("lockupViewModel"); + if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals( + lockupViewModel.getString("contentType"))) { + collector.commit( + new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel)); + } } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 1c19bbda9..d2ea94336 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -726,6 +726,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } else if (result.has("compactPlaylistRenderer")) { return new YoutubeMixOrPlaylistInfoItemExtractor( result.getObject("compactPlaylistRenderer")); + } else if (result.has("lockupViewModel")) { + final JsonObject lockupViewModel = result.getObject("lockupViewModel"); + if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals( + lockupViewModel.getString("contentType"))) { + return new YoutubeMixOrPlaylistLockupInfoItemExtractor( + lockupViewModel); + } } return null; })