be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Fix serialization of Videos channel tab when already fetched 

Also remove usage of Optional as fields as it is not a good practice. This
simplifies in some places channel info extraction code.
This commit is contained in:
AudricV 2024-09-15 15:44:11 +02:00
parent 596bce294d
commit 42c1afaf87
No known key found for this signature in database
GPG Key ID: DA92EC7905614198
3 changed files with 113 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Optional; import java.util.Optional;
@ -233,7 +234,7 @@ public final class YoutubeChannelHelper {
* properties. * properties.
* </p> * </p>
*/ */
public static final class ChannelHeader { public static final class ChannelHeader implements Serializable {
/** /**
* Types of supported YouTube channel headers. * Types of supported YouTube channel headers.
@ -294,27 +295,27 @@ public final class YoutubeChannelHelper {
*/ */
public final HeaderType headerType; public final HeaderType headerType;
private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) { public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
this.json = json; this.json = json;
this.headerType = headerType; this.headerType = headerType;
} }
} }
/** /**
* Get a channel header as an {@link Optional} it if exists. * Get a channel header it if exists.
* *
* @param channelResponse a full channel JSON response * @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional} * @return a {@link ChannelHeader} or {@code null} if no supported header has been found
* if no supported header has been found
*/ */
@Nonnull @Nullable
public static Optional<ChannelHeader> getChannelHeader( public static ChannelHeader getChannelHeader(
@Nonnull final JsonObject channelResponse) { @Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject(HEADER); final JsonObject header = channelResponse.getObject(HEADER);
if (header.has(C4_TABBED_HEADER_RENDERER)) { if (header.has(C4_TABBED_HEADER_RENDERER)) {
return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER)) return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED)); .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
.orElse(null);
} else if (header.has(CAROUSEL_HEADER_RENDERER)) { } else if (header.has(CAROUSEL_HEADER_RENDERER)) {
return header.getObject(CAROUSEL_HEADER_RENDERER) return header.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS) .getArray(CONTENTS)
@ -324,17 +325,20 @@ public final class YoutubeChannelHelper {
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER)) .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst() .findFirst()
.map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER)) .map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL)); .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL))
.orElse(null);
} else if (header.has("pageHeaderRenderer")) { } else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer")) return Optional.of(header.getObject("pageHeaderRenderer"))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE)); .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE))
.orElse(null);
} else if (header.has("interactiveTabbedHeaderRenderer")) { } else if (header.has("interactiveTabbedHeaderRenderer")) {
return Optional.of(header.getObject("interactiveTabbedHeaderRenderer")) return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, .map(json -> new ChannelHeader(json,
ChannelHeader.HeaderType.INTERACTIVE_TABBED)); ChannelHeader.HeaderType.INTERACTIVE_TABBED))
} else { .orElse(null);
return Optional.empty();
} }
return null;
} }
/** /**
@ -418,7 +422,7 @@ public final class YoutubeChannelHelper {
* If the ID cannot still be get, the fallback channel ID, if provided, will be used. * If the ID cannot still be get, the fallback channel ID, if provided, will be used.
* </p> * </p>
* *
* @param header the channel header * @param channelHeader the channel header
* @param fallbackChannelId the fallback channel ID, which can be null * @param fallbackChannelId the fallback channel ID, which can be null
* @return the ID of the channel * @return the ID of the channel
* @throws ParsingException if the channel ID cannot be got from the channel header, the * @throws ParsingException if the channel ID cannot be got from the channel header, the
@ -426,12 +430,10 @@ public final class YoutubeChannelHelper {
*/ */
@Nonnull @Nonnull
public static String getChannelId( public static String getChannelId(
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @Nullable final ChannelHeader channelHeader,
@Nonnull final Optional<ChannelHeader> header,
@Nonnull final JsonObject jsonResponse, @Nonnull final JsonObject jsonResponse,
@Nullable final String fallbackChannelId) throws ParsingException { @Nullable final String fallbackChannelId) throws ParsingException {
if (header.isPresent()) { if (channelHeader != null) {
final ChannelHeader channelHeader = header.get();
switch (channelHeader.headerType) { switch (channelHeader.headerType) {
case C4_TABBED: case C4_TABBED:
final String channelId = channelHeader.json.getObject(HEADER) final String channelId = channelHeader.json.getObject(HEADER)
@ -486,10 +488,9 @@ public final class YoutubeChannelHelper {
} }
@Nonnull @Nonnull
public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static String getChannelName(@Nullable final ChannelHeader channelHeader,
@Nonnull final Optional<ChannelHeader> channelHeader, @Nullable final JsonObject channelAgeGateRenderer,
@Nonnull final JsonObject jsonResponse, @Nonnull final JsonObject jsonResponse)
@Nullable final JsonObject channelAgeGateRenderer)
throws ParsingException { throws ParsingException {
if (channelAgeGateRenderer != null) { if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle"); final String title = channelAgeGateRenderer.getString("channelTitle");
@ -506,7 +507,8 @@ public final class YoutubeChannelHelper {
return metadataRendererTitle; return metadataRendererTitle;
} }
return channelHeader.map(header -> { return Optional.ofNullable(channelHeader)
.map(header -> {
final JsonObject channelJson = header.json; final JsonObject channelJson = header.json;
switch (header.headerType) { switch (header.headerType) {
case PAGE: case PAGE:

133
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -73,8 +73,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject jsonResponse; private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @Nullable
private Optional<ChannelHeader> channelHeader; private ChannelHeader channelHeader;
private String channelId; private String channelId;
@ -132,7 +132,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getName() throws ParsingException { public String getName() throws ParsingException {
assertPageFetched(); assertPageFetched();
return YoutubeChannelHelper.getChannelName( return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse, channelAgeGateRenderer); channelHeader, channelAgeGateRenderer, jsonResponse);
} }
@Nonnull @Nonnull
@ -146,40 +146,40 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.orElseThrow(() -> new ParsingException("Could not get avatars")); .orElseThrow(() -> new ParsingException("Could not get avatars"));
} }
return channelHeader.map(header -> { return Optional.ofNullable(channelHeader)
switch (header.headerType) { .map(header -> {
case PAGE: switch (header.headerType) {
final JsonObject imageObj = header.json.getObject(CONTENT) case PAGE:
.getObject(PAGE_HEADER_VIEW_MODEL) final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(IMAGE); .getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);
if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) { if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL) return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE) .getObject(IMAGE)
.getArray(SOURCES); .getArray(SOURCES);
}
if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
} }
})
if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray) .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElseThrow(() -> new ParsingException("Could not get avatars")); .orElseThrow(() -> new ParsingException("Could not get avatars"));
} }
@ -192,7 +192,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return List.of(); return List.of();
} }
return channelHeader.map(header -> { return Optional.ofNullable(channelHeader)
.map(header -> {
if (header.headerType == HeaderType.PAGE) { if (header.headerType == HeaderType.PAGE) {
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT) final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL); .getObject(PAGE_HEADER_VIEW_MODEL);
@ -235,16 +236,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return UNKNOWN_SUBSCRIBER_COUNT; return UNKNOWN_SUBSCRIBER_COUNT;
} }
if (channelHeader.isPresent()) { if (channelHeader != null) {
final ChannelHeader header = channelHeader.get(); if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
// No subscriber count is available on interactiveTabbedHeaderRenderer header // No subscriber count is available on interactiveTabbedHeaderRenderer header
return UNKNOWN_SUBSCRIBER_COUNT; return UNKNOWN_SUBSCRIBER_COUNT;
} }
final JsonObject headerJson = header.json; final JsonObject headerJson = channelHeader.json;
if (header.headerType == HeaderType.PAGE) { if (channelHeader.headerType == HeaderType.PAGE) {
return getSubscriberCountFromPageChannelHeader(headerJson); return getSubscriberCountFromPageChannelHeader(headerJson);
} }
@ -321,19 +320,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
try { try {
if (channelHeader.isPresent()) { if (channelHeader != null
final ChannelHeader header = channelHeader.get(); && channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
if (header.headerType == HeaderType.INTERACTIVE_TABBED) { /*
/* In an interactiveTabbedHeaderRenderer, the real description, is only available
In an interactiveTabbedHeaderRenderer, the real description, is only available in its header
in its header The other one returned in non-About tabs accessible in the
The other one returned in non-About tabs accessible in the microformatDataRenderer object of the response may be completely different
microformatDataRenderer object of the response may be completely different The description extracted is incomplete and the original one can be only
The description extracted is incomplete and the original one can be only accessed from the About tab
accessed from the About tab */
*/ return getTextFromObject(channelHeader.json.getObject("description"));
return getTextFromObject(header.json.getObject("description"));
}
} }
return jsonResponse.getObject(METADATA) return jsonResponse.getObject(METADATA)
@ -368,8 +365,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return false; return false;
} }
return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() -> if (channelHeader == null) {
new ParsingException("Could not get verified status"))); throw new ParsingException(
"Could not get channel verified status, no channel header has been extracted");
}
return YoutubeChannelHelper.isChannelVerified(channelHeader);
} }
@Nonnull @Nonnull
@ -421,6 +422,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String urlSuffix = urlParts[urlParts.length - 1]; final String urlSuffix = urlParts[urlParts.length - 1];
/*
Make a copy of the channelHeader member to avoid keeping a reference to
this YoutubeChannelExtractor instance which would prevent serialization of
the ReadyChannelTabListLinkHandler instance created above
*/
final ChannelHeader channelHeaderCopy;
if (channelHeader == null) {
channelHeaderCopy = null;
} else {
channelHeaderCopy = new ChannelHeader(channelHeader.json,
channelHeader.headerType);
}
switch (urlSuffix) { switch (urlSuffix) {
case "videos": case "videos":
// Since the Videos tab has already its contents fetched, make // Since the Videos tab has already its contents fetched, make
@ -431,9 +445,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
channelId, channelId,
ChannelTabs.VIDEOS, ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor( (service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, channelHeader, service, linkHandler, tabRenderer,
name, id, url))); channelHeaderCopy, name, id, url)));
break; break;
case "shorts": case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS); addNonVideosTab.accept(ChannelTabs.SHORTS);

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -42,10 +42,11 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
*/ */
public class YoutubeChannelTabExtractor extends ChannelTabExtractor { public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@Nullable
protected YoutubeChannelHelper.ChannelHeader channelHeader;
private JsonObject jsonResponse; private JsonObject jsonResponse;
private String channelId; private String channelId;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
protected Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
public YoutubeChannelTabExtractor(final StreamingService service, public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) { final ListLinkHandler linkHandler) {
@ -104,9 +105,9 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
} }
protected String getChannelName() throws ParsingException { protected String getChannelName() throws ParsingException {
return YoutubeChannelHelper.getChannelName( return YoutubeChannelHelper.getChannelName(channelHeader,
channelHeader, jsonResponse, YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse),
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse)); jsonResponse);
} }
@Nonnull @Nonnull
@ -140,11 +141,14 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
} }
} }
final VerifiedStatus verifiedStatus = channelHeader.flatMap(header -> final VerifiedStatus verifiedStatus;
YoutubeChannelHelper.isChannelVerified(header) if (channelHeader == null) {
? Optional.of(VerifiedStatus.VERIFIED) verifiedStatus = VerifiedStatus.UNKNOWN;
: Optional.of(VerifiedStatus.UNVERIFIED)) } else {
.orElse(VerifiedStatus.UNKNOWN); verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader)
? VerifiedStatus.VERIFIED
: VerifiedStatus.UNVERIFIED;
}
// If a channel tab is fetched, the next page requires channel ID and name, as channel // If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified. // streams don't have their channel specified.
@ -462,8 +466,7 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
VideosTabExtractor(final StreamingService service, VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler, final ListLinkHandler linkHandler,
final JsonObject tabRenderer, final JsonObject tabRenderer,
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @Nullable final YoutubeChannelHelper.ChannelHeader channelHeader,
final Optional<YoutubeChannelHelper.ChannelHeader> channelHeader,
final String channelName, final String channelName,
final String channelId, final String channelId,
final String channelUrl) { final String channelUrl) {