From 42c1afaf873bc7094b3644e32af6aa4fa39fcf2d Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sun, 15 Sep 2024 15:44:11 +0200 Subject: [PATCH] [YouTube] Fix serialization of Videos channel tab when already fetched Also remove usage of Optional as fields as it is not a good practice. This simplifies in some places channel info extraction code. --- .../youtube/YoutubeChannelHelper.java | 48 ++++--- .../extractors/YoutubeChannelExtractor.java | 133 ++++++++++-------- .../YoutubeChannelTabExtractor.java | 27 ++-- 3 files changed, 113 insertions(+), 95 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index 74335017a..54d29032e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -233,7 +234,7 @@ public final class YoutubeChannelHelper { * properties. *

*/ - public static final class ChannelHeader { + public static final class ChannelHeader implements Serializable { /** * Types of supported YouTube channel headers. @@ -294,27 +295,27 @@ public final class YoutubeChannelHelper { */ public final HeaderType headerType; - private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) { + public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) { this.json = json; this.headerType = headerType; } } /** - * Get a channel header as an {@link Optional} it if exists. + * Get a channel header it if exists. * * @param channelResponse a full channel JSON response - * @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional} - * if no supported header has been found + * @return a {@link ChannelHeader} or {@code null} if no supported header has been found */ - @Nonnull - public static Optional getChannelHeader( + @Nullable + public static ChannelHeader getChannelHeader( @Nonnull final JsonObject channelResponse) { final JsonObject header = channelResponse.getObject(HEADER); if (header.has(C4_TABBED_HEADER_RENDERER)) { return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER)) - .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED)); + .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED)) + .orElse(null); } else if (header.has(CAROUSEL_HEADER_RENDERER)) { return header.getObject(CAROUSEL_HEADER_RENDERER) .getArray(CONTENTS) @@ -324,17 +325,20 @@ public final class YoutubeChannelHelper { .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER)) .findFirst() .map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER)) - .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL)); + .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL)) + .orElse(null); } else if (header.has("pageHeaderRenderer")) { return Optional.of(header.getObject("pageHeaderRenderer")) - .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE)); + .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE)) + .orElse(null); } else if (header.has("interactiveTabbedHeaderRenderer")) { return Optional.of(header.getObject("interactiveTabbedHeaderRenderer")) .map(json -> new ChannelHeader(json, - ChannelHeader.HeaderType.INTERACTIVE_TABBED)); - } else { - return Optional.empty(); + ChannelHeader.HeaderType.INTERACTIVE_TABBED)) + .orElse(null); } + + return null; } /** @@ -418,7 +422,7 @@ public final class YoutubeChannelHelper { * If the ID cannot still be get, the fallback channel ID, if provided, will be used. *

* - * @param header the channel header + * @param channelHeader the channel header * @param fallbackChannelId the fallback channel ID, which can be null * @return the ID of the channel * @throws ParsingException if the channel ID cannot be got from the channel header, the @@ -426,12 +430,10 @@ public final class YoutubeChannelHelper { */ @Nonnull public static String getChannelId( - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - @Nonnull final Optional header, + @Nullable final ChannelHeader channelHeader, @Nonnull final JsonObject jsonResponse, @Nullable final String fallbackChannelId) throws ParsingException { - if (header.isPresent()) { - final ChannelHeader channelHeader = header.get(); + if (channelHeader != null) { switch (channelHeader.headerType) { case C4_TABBED: final String channelId = channelHeader.json.getObject(HEADER) @@ -486,10 +488,9 @@ public final class YoutubeChannelHelper { } @Nonnull - public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType") - @Nonnull final Optional channelHeader, - @Nonnull final JsonObject jsonResponse, - @Nullable final JsonObject channelAgeGateRenderer) + public static String getChannelName(@Nullable final ChannelHeader channelHeader, + @Nullable final JsonObject channelAgeGateRenderer, + @Nonnull final JsonObject jsonResponse) throws ParsingException { if (channelAgeGateRenderer != null) { final String title = channelAgeGateRenderer.getString("channelTitle"); @@ -506,7 +507,8 @@ public final class YoutubeChannelHelper { return metadataRendererTitle; } - return channelHeader.map(header -> { + return Optional.ofNullable(channelHeader) + .map(header -> { final JsonObject channelJson = header.json; switch (header.headerType) { case PAGE: diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 50ebb96a3..adc4d948f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -73,8 +73,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private JsonObject jsonResponse; - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private Optional channelHeader; + @Nullable + private ChannelHeader channelHeader; private String channelId; @@ -132,7 +132,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public String getName() throws ParsingException { assertPageFetched(); return YoutubeChannelHelper.getChannelName( - channelHeader, jsonResponse, channelAgeGateRenderer); + channelHeader, channelAgeGateRenderer, jsonResponse); } @Nonnull @@ -146,40 +146,40 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .orElseThrow(() -> new ParsingException("Could not get avatars")); } - return channelHeader.map(header -> { - switch (header.headerType) { - case PAGE: - final JsonObject imageObj = header.json.getObject(CONTENT) - .getObject(PAGE_HEADER_VIEW_MODEL) - .getObject(IMAGE); + return Optional.ofNullable(channelHeader) + .map(header -> { + switch (header.headerType) { + case PAGE: + final JsonObject imageObj = header.json.getObject(CONTENT) + .getObject(PAGE_HEADER_VIEW_MODEL) + .getObject(IMAGE); - if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) { - return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL) - .getObject(IMAGE) - .getArray(SOURCES); + if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) { + return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL) + .getObject(IMAGE) + .getArray(SOURCES); + } + + if (imageObj.has("decoratedAvatarViewModel")) { + return imageObj.getObject("decoratedAvatarViewModel") + .getObject(AVATAR) + .getObject("avatarViewModel") + .getObject(IMAGE) + .getArray(SOURCES); + } + + // Return an empty avatar array as a fallback + return new JsonArray(); + case INTERACTIVE_TABBED: + return header.json.getObject("boxArt") + .getArray(THUMBNAILS); + case C4_TABBED: + case CAROUSEL: + default: + return header.json.getObject(AVATAR) + .getArray(THUMBNAILS); } - - if (imageObj.has("decoratedAvatarViewModel")) { - return imageObj.getObject("decoratedAvatarViewModel") - .getObject(AVATAR) - .getObject("avatarViewModel") - .getObject(IMAGE) - .getArray(SOURCES); - } - - // Return an empty avatar array as a fallback - return new JsonArray(); - case INTERACTIVE_TABBED: - return header.json.getObject("boxArt") - .getArray(THUMBNAILS); - - case C4_TABBED: - case CAROUSEL: - default: - return header.json.getObject(AVATAR) - .getArray(THUMBNAILS); - } - }) + }) .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) .orElseThrow(() -> new ParsingException("Could not get avatars")); } @@ -192,7 +192,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return List.of(); } - return channelHeader.map(header -> { + return Optional.ofNullable(channelHeader) + .map(header -> { if (header.headerType == HeaderType.PAGE) { final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT) .getObject(PAGE_HEADER_VIEW_MODEL); @@ -235,16 +236,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return UNKNOWN_SUBSCRIBER_COUNT; } - if (channelHeader.isPresent()) { - final ChannelHeader header = channelHeader.get(); - - if (header.headerType == HeaderType.INTERACTIVE_TABBED) { + if (channelHeader != null) { + if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) { // No subscriber count is available on interactiveTabbedHeaderRenderer header return UNKNOWN_SUBSCRIBER_COUNT; } - final JsonObject headerJson = header.json; - if (header.headerType == HeaderType.PAGE) { + final JsonObject headerJson = channelHeader.json; + if (channelHeader.headerType == HeaderType.PAGE) { return getSubscriberCountFromPageChannelHeader(headerJson); } @@ -321,19 +320,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } try { - if (channelHeader.isPresent()) { - final ChannelHeader header = channelHeader.get(); - if (header.headerType == HeaderType.INTERACTIVE_TABBED) { - /* - In an interactiveTabbedHeaderRenderer, the real description, is only available - in its header - The other one returned in non-About tabs accessible in the - microformatDataRenderer object of the response may be completely different - The description extracted is incomplete and the original one can be only - accessed from the About tab - */ - return getTextFromObject(header.json.getObject("description")); - } + if (channelHeader != null + && channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) { + /* + In an interactiveTabbedHeaderRenderer, the real description, is only available + in its header + The other one returned in non-About tabs accessible in the + microformatDataRenderer object of the response may be completely different + The description extracted is incomplete and the original one can be only + accessed from the About tab + */ + return getTextFromObject(channelHeader.json.getObject("description")); } return jsonResponse.getObject(METADATA) @@ -368,8 +365,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return false; } - return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() -> - new ParsingException("Could not get verified status"))); + if (channelHeader == null) { + throw new ParsingException( + "Could not get channel verified status, no channel header has been extracted"); + } + + return YoutubeChannelHelper.isChannelVerified(channelHeader); } @Nonnull @@ -421,6 +422,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String urlSuffix = urlParts[urlParts.length - 1]; + /* + Make a copy of the channelHeader member to avoid keeping a reference to + this YoutubeChannelExtractor instance which would prevent serialization of + the ReadyChannelTabListLinkHandler instance created above + */ + final ChannelHeader channelHeaderCopy; + if (channelHeader == null) { + channelHeaderCopy = null; + } else { + channelHeaderCopy = new ChannelHeader(channelHeader.json, + channelHeader.headerType); + } + switch (urlSuffix) { case "videos": // Since the Videos tab has already its contents fetched, make @@ -431,9 +445,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { channelId, ChannelTabs.VIDEOS, (service, linkHandler) -> new VideosTabExtractor( - service, linkHandler, tabRenderer, channelHeader, - name, id, url))); - + service, linkHandler, tabRenderer, + channelHeaderCopy, name, id, url))); break; case "shorts": addNonVideosTab.accept(ChannelTabs.SHORTS); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 6579d09a4..9bd4b3398 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -42,10 +42,11 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; */ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { + @Nullable + protected YoutubeChannelHelper.ChannelHeader channelHeader; + private JsonObject jsonResponse; private String channelId; - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - protected Optional channelHeader; public YoutubeChannelTabExtractor(final StreamingService service, final ListLinkHandler linkHandler) { @@ -104,9 +105,9 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { } protected String getChannelName() throws ParsingException { - return YoutubeChannelHelper.getChannelName( - channelHeader, jsonResponse, - YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse)); + return YoutubeChannelHelper.getChannelName(channelHeader, + YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse), + jsonResponse); } @Nonnull @@ -140,11 +141,14 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { } } - final VerifiedStatus verifiedStatus = channelHeader.flatMap(header -> - YoutubeChannelHelper.isChannelVerified(header) - ? Optional.of(VerifiedStatus.VERIFIED) - : Optional.of(VerifiedStatus.UNVERIFIED)) - .orElse(VerifiedStatus.UNKNOWN); + final VerifiedStatus verifiedStatus; + if (channelHeader == null) { + verifiedStatus = VerifiedStatus.UNKNOWN; + } else { + verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader) + ? VerifiedStatus.VERIFIED + : VerifiedStatus.UNVERIFIED; + } // If a channel tab is fetched, the next page requires channel ID and name, as channel // streams don't have their channel specified. @@ -462,8 +466,7 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { VideosTabExtractor(final StreamingService service, final ListLinkHandler linkHandler, final JsonObject tabRenderer, - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - final Optional channelHeader, + @Nullable final YoutubeChannelHelper.ChannelHeader channelHeader, final String channelName, final String channelId, final String channelUrl) {