be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1222 from AudricV/yt_fix-videos-channel-tab-linkhandler-serialization 

[YouTube] Fix serialization of Videos channel tab when it is already fetched
This commit is contained in:
Audric V. 2024-09-29 15:58:43 +02:00 committed by GitHub
parent 596bce294d 42c1afaf87
commit eb30316a36
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 113 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
@ -233,7 +234,7 @@ public final class YoutubeChannelHelper {
* properties.
* </p>
*/
public static final class ChannelHeader {
public static final class ChannelHeader implements Serializable {
/**
* Types of supported YouTube channel headers.
@ -294,27 +295,27 @@ public final class YoutubeChannelHelper {
*/
public final HeaderType headerType;
private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
this.json = json;
this.headerType = headerType;
}
}
/**
* Get a channel header as an {@link Optional} it if exists.
* Get a channel header it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
* @return a {@link ChannelHeader} or {@code null} if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nullable
public static ChannelHeader getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject(HEADER);
if (header.has(C4_TABBED_HEADER_RENDERER)) {
return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
.orElse(null);
} else if (header.has(CAROUSEL_HEADER_RENDERER)) {
return header.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS)
@ -324,17 +325,20 @@ public final class YoutubeChannelHelper {
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst()
.map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL))
.orElse(null);
} else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer"))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE))
.orElse(null);
} else if (header.has("interactiveTabbedHeaderRenderer")) {
return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json,
ChannelHeader.HeaderType.INTERACTIVE_TABBED));
} else {
return Optional.empty();
ChannelHeader.HeaderType.INTERACTIVE_TABBED))
.orElse(null);
}
return null;
}
/**
@ -418,7 +422,7 @@ public final class YoutubeChannelHelper {
* If the ID cannot still be get, the fallback channel ID, if provided, will be used.
* </p>
*
* @param header the channel header
* @param channelHeader the channel header
* @param fallbackChannelId the fallback channel ID, which can be null
* @return the ID of the channel
* @throws ParsingException if the channel ID cannot be got from the channel header, the
@ -426,12 +430,10 @@ public final class YoutubeChannelHelper {
*/
@Nonnull
public static String getChannelId(
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> header,
@Nullable final ChannelHeader channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final String fallbackChannelId) throws ParsingException {
if (header.isPresent()) {
final ChannelHeader channelHeader = header.get();
if (channelHeader != null) {
switch (channelHeader.headerType) {
case C4_TABBED:
final String channelId = channelHeader.json.getObject(HEADER)
@ -486,10 +488,9 @@ public final class YoutubeChannelHelper {
}
@Nonnull
public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final JsonObject channelAgeGateRenderer)
public static String getChannelName(@Nullable final ChannelHeader channelHeader,
@Nullable final JsonObject channelAgeGateRenderer,
@Nonnull final JsonObject jsonResponse)
throws ParsingException {
if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle");
@ -506,7 +507,8 @@ public final class YoutubeChannelHelper {
return metadataRendererTitle;
}
return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:

133
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -73,8 +73,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<ChannelHeader> channelHeader;
@Nullable
private ChannelHeader channelHeader;
private String channelId;
@ -132,7 +132,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
assertPageFetched();
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse, channelAgeGateRenderer);
channelHeader, channelAgeGateRenderer, jsonResponse);
}
@Nonnull
@ -146,40 +146,40 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}
return channelHeader.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);
return Optional.ofNullable(channelHeader)
.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);
if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}
if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
})
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}
@ -192,7 +192,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return List.of();
}
return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
if (header.headerType == HeaderType.PAGE) {
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
@ -235,16 +236,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return UNKNOWN_SUBSCRIBER_COUNT;
}
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
if (channelHeader != null) {
if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
// No subscriber count is available on interactiveTabbedHeaderRenderer header
return UNKNOWN_SUBSCRIBER_COUNT;
}
final JsonObject headerJson = header.json;
if (header.headerType == HeaderType.PAGE) {
final JsonObject headerJson = channelHeader.json;
if (channelHeader.headerType == HeaderType.PAGE) {
return getSubscriberCountFromPageChannelHeader(headerJson);
}
@ -321,19 +320,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
try {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(header.json.getObject("description"));
}
if (channelHeader != null
&& channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(channelHeader.json.getObject("description"));
}
return jsonResponse.getObject(METADATA)
@ -368,8 +365,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return false;
}
return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() ->
new ParsingException("Could not get verified status")));
if (channelHeader == null) {
throw new ParsingException(
"Could not get channel verified status, no channel header has been extracted");
}
return YoutubeChannelHelper.isChannelVerified(channelHeader);
}
@Nonnull
@ -421,6 +422,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String urlSuffix = urlParts[urlParts.length - 1];
/*
Make a copy of the channelHeader member to avoid keeping a reference to
this YoutubeChannelExtractor instance which would prevent serialization of
the ReadyChannelTabListLinkHandler instance created above
*/
final ChannelHeader channelHeaderCopy;
if (channelHeader == null) {
channelHeaderCopy = null;
} else {
channelHeaderCopy = new ChannelHeader(channelHeader.json,
channelHeader.headerType);
}
switch (urlSuffix) {
case "videos":
// Since the Videos tab has already its contents fetched, make
@ -431,9 +445,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, channelHeader,
name, id, url)));
service, linkHandler, tabRenderer,
channelHeaderCopy, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -42,10 +42,11 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@Nullable
protected YoutubeChannelHelper.ChannelHeader channelHeader;
private JsonObject jsonResponse;
private String channelId;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
protected Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
@ -104,9 +105,9 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
}
protected String getChannelName() throws ParsingException {
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse));
return YoutubeChannelHelper.getChannelName(channelHeader,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse),
jsonResponse);
}
@Nonnull
@ -140,11 +141,14 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
}
}
final VerifiedStatus verifiedStatus = channelHeader.flatMap(header ->
YoutubeChannelHelper.isChannelVerified(header)
? Optional.of(VerifiedStatus.VERIFIED)
: Optional.of(VerifiedStatus.UNVERIFIED))
.orElse(VerifiedStatus.UNKNOWN);
final VerifiedStatus verifiedStatus;
if (channelHeader == null) {
verifiedStatus = VerifiedStatus.UNKNOWN;
} else {
verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader)
? VerifiedStatus.VERIFIED
: VerifiedStatus.UNVERIFIED;
}
// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
@ -462,8 +466,7 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
final Optional<YoutubeChannelHelper.ChannelHeader> channelHeader,
@Nullable final YoutubeChannelHelper.ChannelHeader channelHeader,
final String channelName,
final String channelId,
final String channelUrl) {