be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1240 from AudricV/yt_fix-playlists-items-extraction 

[YouTube] Add support for new playlist items data structure
This commit is contained in:
Stypox 2024-11-14 16:37:35 +01:00 committed by GitHub
parent 0de224124b 183563cc9e
commit d3d5f2b3f0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 273 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View File

@ -18,6 +18,7 @@ import java.util.Optional;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -363,27 +364,11 @@ public final class YoutubeChannelHelper {
final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
final boolean hasCircleOrMusicIcon = pageHeaderViewModel.getObject(TITLE)
final boolean hasCircleOrMusicIcon = hasArtistOrVerifiedIconBadgeAttachment(
pageHeaderViewModel.getObject(TITLE)
.getObject("dynamicTextViewModel")
.getObject("text")
.getArray("attachmentRuns")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
.getObject("type")
.getObject("imageType")
.getObject("image")
.getArray("sources")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(source -> {
final String imageName = source.getObject("clientResource")
.getString("imageName");
return "CHECK_CIRCLE_FILLED".equals(imageName)
|| "MUSIC_FILLED".equals(imageName);
}));
.getArray("attachmentRuns"));
if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image")
.has("contentPreviewImageViewModel")) {
// If a pageHeaderRenderer has no object in which a check verified may be

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1584,6 +1584,29 @@ public final class YoutubeParsingHelper {
return false;
}
public static boolean hasArtistOrVerifiedIconBadgeAttachment(
@Nonnull final JsonArray attachmentRuns) {
return attachmentRuns.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
.getObject("type")
.getObject("imageType")
.getObject("image")
.getArray("sources")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(source -> {
final String imageName = source.getObject("clientResource")
.getString("imageName");
return "CHECK_CIRCLE_FILLED".equals(imageName)
|| "AUDIO_BADGE".equals(imageName)
|| "MUSIC_FILLED".equals(imageName);
}));
}
/**
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
* playback requests (and also for some clients, in the player request body).

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -311,6 +311,12 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("lockupViewModel")) {
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) {
commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus,
channelName, channelUrl);
}
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
@ -366,6 +372,37 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
});
}
private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject playlistLockupViewModel,
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
collector.commit(
new YoutubeMixOrPlaylistLockupInfoItemExtractor(playlistLockupViewModel) {
@Override
public String getUploaderName() throws ParsingException {
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
}
@Override
public String getUploaderUrl() throws ParsingException {
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
switch (channelVerifiedStatus) {
case VERIFIED:
return true;
case UNVERIFIED:
return false;
default:
return super.isUploaderVerified();
}
}
});
}
private void commitVideo(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final JsonObject jsonObject,

193
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java Normal file
View File

@ -0,0 +1,193 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Image;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
public class YoutubeMixOrPlaylistLockupInfoItemExtractor implements PlaylistInfoItemExtractor {
@Nonnull
private final JsonObject lockupViewModel;
@Nonnull
private final JsonObject thumbnailViewModel;
@Nonnull
private final JsonObject lockupMetadataViewModel;
@Nonnull
private final JsonObject firstMetadataRow;
@Nonnull
private PlaylistInfo.PlaylistType playlistType;
public YoutubeMixOrPlaylistLockupInfoItemExtractor(@Nonnull final JsonObject lockupViewModel) {
this.lockupViewModel = lockupViewModel;
this.thumbnailViewModel = lockupViewModel.getObject("contentImage")
.getObject("collectionThumbnailViewModel")
.getObject("primaryThumbnail")
.getObject("thumbnailViewModel");
this.lockupMetadataViewModel = lockupViewModel.getObject("metadata")
.getObject("lockupMetadataViewModel");
/*
The metadata rows are structured in the following way:
1st part: uploader info, playlist type, playlist updated date
2nd part: space row
3rd element: first video
4th (not always returned for playlists with less than 2 items?): second video
5th element (always returned, but at a different index for playlists with less than 2
items?): Show full playlist
The first metadata row has the following structure:
1st array element: uploader info
2nd element: playlist type (course, playlist, podcast)
3rd element (not always returned): playlist updated date
*/
this.firstMetadataRow = lockupMetadataViewModel.getObject("metadata")
.getObject("contentMetadataViewModel")
.getArray("metadataRows")
.getObject(0);
try {
this.playlistType = extractPlaylistTypeFromPlaylistId(getPlaylistId());
} catch (final ParsingException e) {
// If we cannot extract the playlist type, fall back to the normal one
this.playlistType = PlaylistInfo.PlaylistType.NORMAL;
}
}
@Override
public String getUploaderName() throws ParsingException {
return firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getString("content");
}
@Override
public String getUploaderUrl() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, there is no uploader as they are auto-generated
return null;
}
return getUrlFromNavigationEndpoint(
firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getArray("commandRuns")
.getObject(0)
.getObject("onTap")
.getObject("innertubeCommand"));
}
@Override
public boolean isUploaderVerified() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, there is no uploader as they are auto-generated
return false;
}
return hasArtistOrVerifiedIconBadgeAttachment(
firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getArray("attachmentRuns"));
}
@Override
public long getStreamCount() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, we are not able to get its stream count
return ListExtractor.ITEM_COUNT_INFINITE;
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
thumbnailViewModel.getArray("overlays")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(overlay -> overlay.has("thumbnailOverlayBadgeViewModel"))
.findFirst()
.orElseThrow(() -> new ParsingException(
"Could not get thumbnailOverlayBadgeViewModel"))
.getObject("thumbnailOverlayBadgeViewModel")
.getArray("thumbnailBadges")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(badge -> badge.has("thumbnailBadgeViewModel"))
.findFirst()
.orElseThrow(() ->
new ParsingException("Could not get thumbnailBadgeViewModel"))
.getObject("thumbnailBadgeViewModel")
.getString("text")));
} catch (final Exception e) {
throw new ParsingException("Could not get playlist stream count", e);
}
}
@Override
public String getName() throws ParsingException {
return lockupMetadataViewModel.getObject("title")
.getString("content");
}
@Override
public String getUrl() throws ParsingException {
// If the playlist item is a mix, we cannot return just its playlist ID as mix playlists
// are not viewable in playlist pages
// Use directly getUrlFromNavigationEndpoint in this case, which returns the watch URL with
// the mix playlist
if (playlistType == PlaylistInfo.PlaylistType.NORMAL) {
try {
return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(getPlaylistId());
} catch (final Exception ignored) {
}
}
return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext")
.getObject("commandContext")
.getObject("onTap")
.getObject("innertubeCommand"));
}
@Nonnull
@Override
public List<Image> getThumbnails() throws ParsingException {
return getImagesFromThumbnailsArray(thumbnailViewModel.getObject("image")
.getArray("sources"));
}
@Nonnull
@Override
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
return playlistType;
}
private String getPlaylistId() throws ParsingException {
String id = lockupViewModel.getString("contentId");
if (Utils.isNullOrEmpty(id)) {
id = lockupViewModel.getObject("rendererContext")
.getObject("commandContext")
.getObject("watchEndpoint")
.getString("playlistId");
}
if (Utils.isNullOrEmpty(id)) {
throw new ParsingException("Could not get playlist ID");
}
return id;
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -245,6 +245,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
} else if (item.has("showRenderer")) {
collector.commit(new YoutubeShowRendererInfoItemExtractor(
item.getObject("showRenderer")));
} else if (item.has("lockupViewModel")) {
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
lockupViewModel.getString("contentType"))) {
collector.commit(
new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel));
}
}
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -726,6 +726,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} else if (result.has("compactPlaylistRenderer")) {
return new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactPlaylistRenderer"));
} else if (result.has("lockupViewModel")) {
final JsonObject lockupViewModel = result.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
lockupViewModel.getString("contentType"))) {
return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
lockupViewModel);
}
}
return null;
})