be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1203 from AudricV/yt_support-shows-and-pageheader-on-user-channels 

[YouTube] Support shows and page header on user channels
This commit is contained in:
Stypox 2024-07-25 18:03:13 +02:00 committed by GitHub
parent 2d36945b39 8db724943d
commit 996eb046aa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 2454 additions and 2073 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View File

@ -4,16 +4,19 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -21,6 +24,19 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* Shared functions for extracting YouTube channel pages and tabs.
*/
public final class YoutubeChannelHelper {
private static final String BROWSE_ENDPOINT = "browseEndpoint";
private static final String BROWSE_ID = "browseId";
private static final String CAROUSEL_HEADER_RENDERER = "carouselHeaderRenderer";
private static final String C4_TABBED_HEADER_RENDERER = "c4TabbedHeaderRenderer";
private static final String CONTENT = "content";
private static final String CONTENTS = "contents";
private static final String HEADER = "header";
private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel";
private static final String TAB_RENDERER = "tabRenderer";
private static final String TITLE = "title";
private static final String TOPIC_CHANNEL_DETAILS_RENDERER = "topicChannelDetailsRenderer";
private YoutubeChannelHelper() {
}
@ -64,8 +80,8 @@ public final class YoutubeChannelHelper {
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
final JsonObject browseEndpoint = endpoint.getObject(BROWSE_ENDPOINT);
final String browseId = browseEndpoint.getString(BROWSE_ID, "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
@ -140,7 +156,7 @@ public final class YoutubeChannelHelper {
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, country)
.value("browseId", id)
.value(BROWSE_ID, id)
.value("params", parameters)
.done())
.getBytes(StandardCharsets.UTF_8);
@ -159,8 +175,8 @@ public final class YoutubeChannelHelper {
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint")
.getString("browseId", "");
final String browseId = endpoint.getObject(BROWSE_ENDPOINT)
.getString(BROWSE_ID, "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
@ -257,7 +273,7 @@ public final class YoutubeChannelHelper {
* A {@code pageHeaderRenderer} channel header type.
*
* <p>
* This header returns only the channel's name and its avatar.
* This header returns only the channel's name and its avatar for system channels.
* </p>
*/
PAGE
@ -294,20 +310,20 @@ public final class YoutubeChannelHelper {
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject("header");
final JsonObject header = channelResponse.getObject(HEADER);
if (header.has("c4TabbedHeaderRenderer")) {
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
if (header.has(C4_TABBED_HEADER_RENDERER)) {
return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
} else if (header.has("carouselHeaderRenderer")) {
return header.getObject("carouselHeaderRenderer")
.getArray("contents")
} else if (header.has(CAROUSEL_HEADER_RENDERER)) {
return header.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS)
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst()
.map(item -> item.getObject("topicChannelDetailsRenderer"))
.map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
} else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer"))
@ -320,4 +336,221 @@ public final class YoutubeChannelHelper {
return Optional.empty();
}
}
/**
* Check if a channel is verified by using its header.
*
* <p>
* The header is mandatory, so the verified status of age-restricted channels with a
* {@code channelAgeGateRenderer} cannot be checked.
* </p>
*
* @param channelHeader the {@link ChannelHeader} of a non age-restricted channel
* @return whether the channel is verified
*/
public static boolean isChannelVerified(@Nonnull final ChannelHeader channelHeader) {
switch (channelHeader.headerType) {
// carouselHeaderRenderers do not contain any verification badges
// Since they are only shown on YouTube internal channels or on channels of large
// organizations broadcasting live events, we can assume the channel to be verified
case CAROUSEL:
return true;
case PAGE:
final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
final boolean hasCircleOrMusicIcon = pageHeaderViewModel.getObject(TITLE)
.getObject("dynamicTextViewModel")
.getObject("text")
.getArray("attachmentRuns")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
.getObject("type")
.getObject("imageType")
.getObject("image")
.getArray("sources")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(source -> {
final String imageName = source.getObject("clientResource")
.getString("imageName");
return "CHECK_CIRCLE_FILLED".equals(imageName)
|| "MUSIC_FILLED".equals(imageName);
}));
if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image")
.has("contentPreviewImageViewModel")) {
// If a pageHeaderRenderer has no object in which a check verified may be
// contained and if it has a contentPreviewImageViewModel, it should mean
// that the header is coming from a system channel, which we can assume to
// be verified
return true;
}
return hasCircleOrMusicIcon;
case INTERACTIVE_TABBED:
// If the header has an autoGenerated property, it should mean that the channel has
// been auto generated by YouTube: we can assume the channel to be verified in this
// case
return channelHeader.json.has("autoGenerated");
default:
return YoutubeParsingHelper.isVerified(channelHeader.json.getArray("badges"));
}
}
/**
* Get the ID of a channel from its response.
*
* <p>
* For {@link ChannelHeader.HeaderType#C4_TABBED c4TabbedHeaderRenderer} and
* {@link ChannelHeader.HeaderType#CAROUSEL carouselHeaderRenderer} channel headers, the ID is
* get from the header.
* </p>
*
* <p>
* For other headers or if it cannot be got, the ID from the {@code channelMetadataRenderer}
* in the channel response is used.
* </p>
*
* <p>
* If the ID cannot still be get, the fallback channel ID, if provided, will be used.
* </p>
*
* @param header the channel header
* @param fallbackChannelId the fallback channel ID, which can be null
* @return the ID of the channel
* @throws ParsingException if the channel ID cannot be got from the channel header, the
* channel response and the fallback channel ID
*/
@Nonnull
public static String getChannelId(
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> header,
@Nonnull final JsonObject jsonResponse,
@Nullable final String fallbackChannelId) throws ParsingException {
if (header.isPresent()) {
final ChannelHeader channelHeader = header.get();
switch (channelHeader.headerType) {
case C4_TABBED:
final String channelId = channelHeader.json.getObject(HEADER)
.getObject(C4_TABBED_HEADER_RENDERER)
.getString("channelId", "");
if (!isNullOrEmpty(channelId)) {
return channelId;
}
final String navigationC4TabChannelId = channelHeader.json
.getObject("navigationEndpoint")
.getObject(BROWSE_ENDPOINT)
.getString(BROWSE_ID);
if (!isNullOrEmpty(navigationC4TabChannelId)) {
return navigationC4TabChannelId;
}
break;
case CAROUSEL:
final String navigationCarouselChannelId = channelHeader.json.getObject(HEADER)
.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS)
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst()
.orElse(new JsonObject())
.getObject(TOPIC_CHANNEL_DETAILS_RENDERER)
.getObject("navigationEndpoint")
.getObject(BROWSE_ENDPOINT)
.getString(BROWSE_ID);
if (!isNullOrEmpty(navigationCarouselChannelId)) {
return navigationCarouselChannelId;
}
break;
default:
break;
}
}
final String externalChannelId = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("externalChannelId");
if (!isNullOrEmpty(externalChannelId)) {
return externalChannelId;
}
if (!isNullOrEmpty(fallbackChannelId)) {
return fallbackChannelId;
} else {
throw new ParsingException("Could not get channel ID");
}
}
@Nonnull
public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final JsonObject channelAgeGateRenderer)
throws ParsingException {
if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle");
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get channel name");
}
return title;
}
final String metadataRendererTitle = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString(TITLE);
if (!isNullOrEmpty(metadataRendererTitle)) {
return metadataRendererTitle;
}
return channelHeader.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:
return channelJson.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(TITLE)
.getObject("dynamicTextViewModel")
.getObject("text")
.getString(CONTENT, channelJson.getString("pageTitle"));
case CAROUSEL:
case INTERACTIVE_TABBED:
return getTextFromObject(channelJson.getObject(TITLE));
case C4_TABBED:
default:
return channelJson.getString(TITLE);
}
})
// The channel name from a microformatDataRenderer may be different from the one
// displayed, especially for auto-generated channels, depending on the language
// requested for the interface (hl parameter of InnerTube requests' payload)
.or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getString(TITLE)))
.orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Nullable
public static JsonObject getChannelAgeGateRenderer(@Nonnull final JsonObject jsonResponse) {
return jsonResponse.getObject(CONTENTS)
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject(TAB_RENDERER)
.getObject(CONTENT)
.getObject("sectionListRenderer")
.getArray(CONTENTS)
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -825,9 +825,15 @@ public final class YoutubeParsingHelper {
final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
final String browseId = browseEndpoint.getString("browseId");
// All channel ids are prefixed with UC
if (browseId != null && browseId.startsWith("UC")) {
return "https://www.youtube.com/channel/" + browseId;
if (browseId != null) {
if (browseId.startsWith("UC")) {
// All channel IDs are prefixed with UC
return "https://www.youtube.com/channel/" + browseId;
} else if (browseId.startsWith("VL")) {
// All playlist IDs are prefixed with VL, which needs to be removed from the
// playlist ID
return "https://www.youtube.com/playlist?list=" + browseId.substring(2);
}
}
if (!isNullOrEmpty(canonicalBaseUrl)) {
@ -887,12 +893,13 @@ public final class YoutubeParsingHelper {
return textObject.getString("simpleText");
}
if (textObject.getArray("runs").isEmpty()) {
final JsonArray runs = textObject.getArray("runs");
if (runs.isEmpty()) {
return null;
}
final StringBuilder textBuilder = new StringBuilder();
for (final Object o : textObject.getArray("runs")) {
for (final Object o : runs) {
final JsonObject run = (JsonObject) o;
String text = run.getString("text");
@ -970,11 +977,12 @@ public final class YoutubeParsingHelper {
return null;
}
if (textObject.getArray("runs").isEmpty()) {
final JsonArray runs = textObject.getArray("runs");
if (runs.isEmpty()) {
return null;
}
for (final Object textPart : textObject.getArray("runs")) {
for (final Object textPart : runs) {
final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java Normal file
View File

@ -0,0 +1,65 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Image;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/**
* The base {@link PlaylistInfoItemExtractor} for shows playlists UI elements.
*/
abstract class YoutubeBaseShowInfoItemExtractor implements PlaylistInfoItemExtractor {
@Nonnull
protected final JsonObject showRenderer;
YoutubeBaseShowInfoItemExtractor(@Nonnull final JsonObject showRenderer) {
this.showRenderer = showRenderer;
}
@Override
public String getName() throws ParsingException {
return showRenderer.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint"));
}
@Nonnull
@Override
public List<Image> getThumbnails() throws ParsingException {
return getThumbnailsFromInfoItem(showRenderer.getObject("thumbnailRenderer")
.getObject("showCustomThumbnailRenderer"));
}
@Override
public long getStreamCount() throws ParsingException {
// The stream count should be always returned in the first text object for English
// localizations, but the complete text is parsed for reliability purposes
final String streamCountText = getTextFromObject(
showRenderer.getObject("thumbnailOverlays")
.getObject("thumbnailOverlayBottomPanelRenderer")
.getObject("text"));
if (streamCountText == null) {
throw new ParsingException("Could not get stream count");
}
try {
// The data returned could be a human/shortened number, but no show with more than 1000
// videos has been found at the time this code was written
return Long.parseLong(Utils.removeNonDigitCharacters(streamCountText));
} catch (final NumberFormatException e) {
throw new ParsingException("Could not convert stream count to a long", e);
}
}
}

251
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -23,7 +23,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -59,6 +58,19 @@ import javax.annotation.Nullable;
public class YoutubeChannelExtractor extends ChannelExtractor {
// Constants of objects used multiples from channel responses
private static final String IMAGE = "image";
private static final String CONTENTS = "contents";
private static final String CONTENT_PREVIEW_IMAGE_VIEW_MODEL = "contentPreviewImageViewModel";
private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel";
private static final String TAB_RENDERER = "tabRenderer";
private static final String CONTENT = "content";
private static final String METADATA = "metadata";
private static final String AVATAR = "avatar";
private static final String THUMBNAILS = "thumbnails";
private static final String SOURCES = "sources";
private static final String BANNER = "banner";
private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@ -95,28 +107,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
jsonResponse = data.jsonResponse;
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
channelId = data.channelId;
channelAgeGateRenderer = getChannelAgeGateRenderer();
}
@Nullable
private JsonObject getChannelAgeGateRenderer() {
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse);
}
@Nonnull
@ -133,62 +124,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getId() throws ParsingException {
assertPageFetched();
return channelHeader.map(header -> header.json)
.flatMap(header -> Optional.ofNullable(header.getString("channelId"))
.or(() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
.or(() -> Optional.ofNullable(channelId))
.orElseThrow(() -> new ParsingException("Could not get channel ID"));
return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
}
@Nonnull
@Override
public String getName() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle");
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get channel name");
}
return title;
}
final String metadataRendererTitle = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataRendererTitle)) {
return metadataRendererTitle;
}
return channelHeader.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:
return channelJson.getObject("content")
.getObject("pageHeaderViewModel")
.getObject("title")
.getObject("dynamicTextViewModel")
.getObject("text")
.getString("content", channelJson.getString("pageTitle"));
case CAROUSEL:
case INTERACTIVE_TABBED:
return getTextFromObject(channelJson.getObject("title"));
case C4_TABBED:
default:
return channelJson.getString("title");
}
})
// The channel name from a microformatDataRenderer may be different from the one displayed,
// especially for auto-generated channels, depending on the language requested for the
// interface (hl parameter of InnerTube requests' payload)
.or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getString("title")))
.orElseThrow(() -> new ParsingException("Could not get channel name"));
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse, channelAgeGateRenderer);
}
@Nonnull
@ -196,8 +140,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public List<Image> getAvatars() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
.getArray("thumbnails"))
return Optional.ofNullable(channelAgeGateRenderer.getObject(AVATAR)
.getArray(THUMBNAILS))
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}
@ -205,22 +149,35 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return channelHeader.map(header -> {
switch (header.headerType) {
case PAGE:
return header.json.getObject("content")
.getObject("pageHeaderViewModel")
.getObject("image")
.getObject("contentPreviewImageViewModel")
.getObject("image")
.getArray("sources");
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);
if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}
if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray("thumbnails");
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject("avatar")
.getArray("thumbnails");
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
@ -235,10 +192,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return List.of();
}
// No banner is available on pageHeaderRenderer headers
return channelHeader.filter(header -> header.headerType != HeaderType.PAGE)
.map(header -> header.json.getObject("banner")
.getArray("thumbnails"))
return channelHeader.map(header -> {
if (header.headerType == HeaderType.PAGE) {
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
if (pageHeaderViewModel.has(BANNER)) {
return pageHeaderViewModel.getObject(BANNER)
.getObject("imageBannerViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}
// No banner is available (this should happen on pageHeaderRenderers of
// system channels), use an empty JsonArray instead
return new JsonArray();
}
return header.json
.getObject(BANNER)
.getArray(THUMBNAILS);
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElse(List.of());
}
@ -264,14 +238,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED
|| header.headerType == HeaderType.PAGE) {
// No subscriber count is available on interactiveTabbedHeaderRenderer and
// pageHeaderRenderer headers
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
// No subscriber count is available on interactiveTabbedHeaderRenderer header
return UNKNOWN_SUBSCRIBER_COUNT;
}
final JsonObject headerJson = header.json;
if (header.headerType == HeaderType.PAGE) {
return getSubscriberCountFromPageChannelHeader(headerJson);
}
JsonObject textObject = null;
if (headerJson.has("subscriberCountText")) {
@ -292,6 +268,51 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return UNKNOWN_SUBSCRIBER_COUNT;
}
private long getSubscriberCountFromPageChannelHeader(@Nonnull final JsonObject headerJson)
throws ParsingException {
final JsonObject metadataObject = headerJson.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(METADATA);
if (metadataObject.has("contentMetadataViewModel")) {
final JsonArray metadataPart = metadataObject.getObject("contentMetadataViewModel")
.getArray("metadataRows")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(metadataRow -> metadataRow.getArray("metadataParts"))
/*
Find metadata parts which have two elements: channel handle and subscriber
count.
On autogenerated music channels, the subscriber count is not shown with this
header.
Use the first metadata parts object found.
*/
.filter(metadataParts -> metadataParts.size() == 2)
.findFirst()
.orElse(null);
if (metadataPart == null) {
// As the parsing of the metadata parts object needed to get the subscriber count
// is fragile, return UNKNOWN_SUBSCRIBER_COUNT when it cannot be got
return UNKNOWN_SUBSCRIBER_COUNT;
}
try {
// The subscriber count is at the same position for all languages as of 02/03/2024
return Utils.mixedNumberWordToLong(metadataPart.getObject(0)
.getObject("text")
.getString(CONTENT));
} catch (final NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
}
// If the channel header has no contentMetadataViewModel (which is the case for system
// channels using this header), return UNKNOWN_SUBSCRIBER_COUNT
return UNKNOWN_SUBSCRIBER_COUNT;
}
@Override
public String getDescription() throws ParsingException {
assertPageFetched();
@ -302,12 +323,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
try {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.PAGE) {
// A pageHeaderRenderer doesn't contain a description
return null;
}
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
@ -321,8 +336,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
// The description is cut and the original one can be only accessed from the About tab
return jsonResponse.getObject("metadata")
return jsonResponse.getObject(METADATA)
.getObject("channelMetadataRenderer")
.getString("description");
} catch (final Exception e) {
@ -350,31 +364,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public boolean isVerified() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
// Verified status is unknown with channelAgeGateRenderers, return false in this case
return false;
}
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
// carouselHeaderRenderer and pageHeaderRenderer does not contain any verification
// badges
// Since they are only shown on YouTube internal channels or on channels of large
// organizations broadcasting live events, we can assume the channel to be verified
if (header.headerType == HeaderType.CAROUSEL || header.headerType == HeaderType.PAGE) {
return true;
}
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
// If the header has an autoGenerated property, it should mean that the channel has
// been auto generated by YouTube: we can assume the channel to be verified in this
// case
return header.json.has("autoGenerated");
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() ->
new ParsingException("Could not get verified status")));
}
@Nonnull
@ -390,7 +385,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
final JsonArray responseTabs = jsonResponse.getObject("contents")
final JsonArray responseTabs = jsonResponse.getObject(CONTENTS)
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
@ -411,8 +406,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
responseTabs.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.filter(tab -> tab.has(TAB_RENDERER))
.map(tab -> tab.getObject(TAB_RENDERER))
.forEach(tabRenderer -> {
final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata")
@ -436,7 +431,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, name, id, url)));
service, linkHandler, tabRenderer, channelHeader,
name, id, url)));
break;
case "shorts":
@ -451,6 +447,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
case "playlists":
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
break;
default:
// Unsupported channel tab, ignore it
break;
}
}
});

326
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -29,7 +29,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -37,8 +36,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* A {@link ChannelTabExtractor} implementation for the YouTube service.
*
* <p>
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
* {@code Channels} tabs.
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists},
* {@code Albums} and {@code Channels} tabs.
* </p>
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@ -60,6 +59,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
private String channelId;
@Nullable
private String visitorData;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
protected Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
@ -89,14 +90,15 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
channelId = resolveChannelId(super.getId());
final String channelIdFromId = resolveChannelId(super.getId());
final String params = getChannelTabsParameters();
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelIdFromId,
params, getExtractorLocalization(), getExtractorContentCountry());
jsonResponse = data.jsonResponse;
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
channelId = data.channelId;
if (useVisitorData) {
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
@ -117,60 +119,13 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (!id.isEmpty()) {
return id;
}
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
}
if (!isNullOrEmpty(channelId)) {
return channelId;
} else {
throw new ParsingException("Could not get channel ID");
}
return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
}
protected String getChannelName() {
final String metadataName = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataName)) {
return metadataName;
}
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
})
.orElse("");
protected String getChannelName() throws ParsingException {
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse));
}
@Nonnull
@ -204,18 +159,27 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
}
}
final VerifiedStatus verifiedStatus = channelHeader.flatMap(header ->
YoutubeChannelHelper.isChannelVerified(header)
? Optional.of(VerifiedStatus.VERIFIED)
: Optional.of(VerifiedStatus.UNVERIFIED))
.orElse(VerifiedStatus.UNKNOWN);
// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
// We also need to set the visitor data here when it should be enabled, as it is required
// to get continuations on some channel tabs, and we need a way to pass it between pages
final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
? List.of(getChannelName(), getUrl(), visitorData)
: List.of(getChannelName(), getUrl());
final String channelName = getChannelName();
final String channelUrl = getUrl();
final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
final JsonObject continuation = collectItemsFrom(collector, items, verifiedStatus,
channelName, channelUrl)
.orElse(null);
final Page nextPage = getNextPageFrom(continuation, channelIds);
final Page nextPage = getNextPageFrom(continuation,
useVisitorData && !isNullOrEmpty(visitorData)
? List.of(channelName, channelUrl, verifiedStatus.toString(), visitorData)
: List.of(channelName, channelUrl, verifiedStatus.toString()));
return new InfoItemsPage<>(collector, nextPage);
}
@ -281,16 +245,48 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final List<String> channelIds) {
final String channelName;
final String channelUrl;
VerifiedStatus verifiedStatus;
if (channelIds.size() >= 3) {
channelName = channelIds.get(0);
channelUrl = channelIds.get(1);
try {
verifiedStatus = VerifiedStatus.valueOf(channelIds.get(2));
} catch (final IllegalArgumentException e) {
// An IllegalArgumentException can be thrown if someone passes a third channel ID
// which is not of the enum type in the getPage method, use the UNKNOWN
// VerifiedStatus enum value in this case
verifiedStatus = VerifiedStatus.UNKNOWN;
}
} else {
channelName = null;
channelUrl = null;
verifiedStatus = VerifiedStatus.UNKNOWN;
}
return collectItemsFrom(collector, items, verifiedStatus, channelName, channelUrl);
}
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final VerifiedStatus verifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
return items.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(item -> collectItem(collector, item, channelIds))
.map(item -> collectItem(
collector, item, verifiedStatus, channelName, channelUrl))
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
}
private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject item,
@Nonnull final List<String> channelIds) {
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
if (item.has("richItemRenderer")) {
@ -298,33 +294,37 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
.getObject("content");
if (richItem.has("videoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds,
richItem.getObject("videoRenderer"));
commitVideo(collector, timeAgoParser, richItem.getObject("videoRenderer"),
channelVerifiedStatus, channelName, channelUrl);
} else if (richItem.has("reelItemRenderer")) {
getCommitReelItemConsumer(collector, channelIds,
richItem.getObject("reelItemRenderer"));
commitReel(collector, richItem.getObject("reelItemRenderer"),
channelVerifiedStatus, channelName, channelUrl);
} else if (richItem.has("playlistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds,
richItem.getObject("playlistRenderer"));
commitPlaylist(collector, richItem.getObject("playlistRenderer"),
channelVerifiedStatus, channelName, channelUrl);
}
} else if (item.has("gridVideoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds,
item.getObject("gridVideoRenderer"));
commitVideo(collector, timeAgoParser, item.getObject("gridVideoRenderer"),
channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("gridPlaylistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds,
item.getObject("gridPlaylistRenderer"));
commitPlaylist(collector, item.getObject("gridPlaylistRenderer"),
channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("gridShowRenderer")) {
collector.commit(new YoutubeGridShowRendererChannelInfoItemExtractor(
item.getObject("gridShowRenderer"), channelVerifiedStatus, channelName,
channelUrl));
} else if (item.has("shelfRenderer")) {
return collectItem(collector, item.getObject("shelfRenderer")
.getObject("content"), channelIds);
.getObject("content"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("itemSectionRenderer")) {
return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
.getArray("contents"), channelIds);
.getArray("contents"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("horizontalListRenderer")) {
return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
.getArray("items"), channelIds);
.getArray("items"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelIds);
.getArray("items"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
@ -332,72 +332,91 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
return Optional.empty();
}
private void getCommitVideoConsumer(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds,
@Nonnull final JsonObject jsonObject) {
private static void commitReel(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject reelItemRenderer,
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
collector.commit(
new YoutubeReelInfoItemExtractor(reelItemRenderer) {
@Override
public String getUploaderName() throws ParsingException {
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
}
@Override
public String getUploaderUrl() throws ParsingException {
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
}
@Override
public boolean isUploaderVerified() {
return channelVerifiedStatus == VerifiedStatus.VERIFIED;
}
});
}
private void commitVideo(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final JsonObject jsonObject,
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
collector.commit(
new YoutubeStreamInfoItemExtractor(jsonObject, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
}
@SuppressWarnings("DuplicatedCode")
@Override
public boolean isUploaderVerified() throws ParsingException {
switch (channelVerifiedStatus) {
case VERIFIED:
return true;
case UNVERIFIED:
return false;
default:
return super.isUploaderVerified();
}
return super.getUploaderUrl();
}
});
}
private void getCommitReelItemConsumer(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final List<String> channelIds,
@Nonnull final JsonObject jsonObject) {
collector.commit(
new YoutubeReelInfoItemExtractor(jsonObject) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
private void getCommitPlaylistConsumer(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final List<String> channelIds,
@Nonnull final JsonObject jsonObject) {
private void commitPlaylist(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject jsonObject,
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
collector.commit(
new YoutubePlaylistInfoItemExtractor(jsonObject) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
}
@SuppressWarnings("DuplicatedCode")
@Override
public boolean isUploaderVerified() throws ParsingException {
switch (channelVerifiedStatus) {
case VERIFIED:
return true;
case UNVERIFIED:
return false;
default:
return super.isUploaderVerified();
}
return super.getUploaderUrl();
}
});
}
@ -431,20 +450,24 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
*/
public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
private final JsonObject tabRenderer;
private final String channelName;
private final String channelId;
private final String channelName;
private final String channelUrl;
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
final Optional<YoutubeChannelHelper.ChannelHeader> channelHeader,
final String channelName,
final String channelId,
final String channelUrl) {
super(service, linkHandler);
this.channelHeader = channelHeader;
this.tabRenderer = tabRenderer;
this.channelName = channelName;
this.channelId = channelId;
this.channelName = channelName;
this.channelUrl = channelUrl;
}
@ -475,4 +498,59 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
return Optional.of(tabRenderer);
}
}
/**
* Enum representing the verified state of a channel
*/
private enum VerifiedStatus {
VERIFIED,
UNVERIFIED,
UNKNOWN
}
private static final class YoutubeGridShowRendererChannelInfoItemExtractor
extends YoutubeBaseShowInfoItemExtractor {
@Nonnull
private final VerifiedStatus verifiedStatus;
@Nullable
private final String channelName;
@Nullable
private final String channelUrl;
private YoutubeGridShowRendererChannelInfoItemExtractor(
@Nonnull final JsonObject gridShowRenderer,
@Nonnull final VerifiedStatus verifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
super(gridShowRenderer);
this.verifiedStatus = verifiedStatus;
this.channelName = channelName;
this.channelUrl = channelUrl;
}
@Override
public String getUploaderName() {
return channelName;
}
@Override
public String getUploaderUrl() {
return channelUrl;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
switch (verifiedStatus) {
case VERIFIED:
return true;
case UNVERIFIED:
return false;
default:
throw new ParsingException("Could not get uploader verification status");
}
}
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -238,9 +238,14 @@ public class YoutubeSearchExtractor extends SearchExtractor {
} else if (extractChannelResults && item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("channelRenderer")));
} else if (extractPlaylistResults && item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(
item.getObject("playlistRenderer")));
} else if (extractPlaylistResults) {
if (item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(
item.getObject("playlistRenderer")));
} else if (item.has("showRenderer")) {
collector.commit(new YoutubeShowRendererInfoItemExtractor(
item.getObject("showRenderer")));
}
}
}
}

57
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java Normal file
View File

@ -0,0 +1,57 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link YoutubeBaseShowInfoItemExtractor} implementation for {@code showRenderer}s.
*/
class YoutubeShowRendererInfoItemExtractor extends YoutubeBaseShowInfoItemExtractor {
@Nonnull
private final JsonObject shortBylineText;
@Nonnull
private final JsonObject longBylineText;
YoutubeShowRendererInfoItemExtractor(@Nonnull final JsonObject showRenderer) {
super(showRenderer);
this.shortBylineText = showRenderer.getObject("shortBylineText");
this.longBylineText = showRenderer.getObject("longBylineText");
}
@Override
public String getUploaderName() throws ParsingException {
String name = getTextFromObject(longBylineText);
if (isNullOrEmpty(name)) {
name = getTextFromObject(shortBylineText);
if (isNullOrEmpty(name)) {
throw new ParsingException("Could not get uploader name");
}
}
return name;
}
@Override
public String getUploaderUrl() throws ParsingException {
String uploaderUrl = getUrlFromObject(longBylineText);
if (uploaderUrl == null) {
uploaderUrl = getUrlFromObject(shortBylineText);
if (uploaderUrl == null) {
throw new ParsingException("Could not get uploader URL");
}
}
return uploaderUrl;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
// We do not have this information in showRenderers
return false;
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -294,7 +294,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testDescription() throws Exception {
assertContains("Our World is Amazing. \n\nQuestions? Ideas? Tweet me:", extractor.getDescription());
assertContains("Our World is Amazing", extractor.getDescription());
}
@Test

8
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/shorts/generated_mock_0.json
View File

@ -41,10 +41,10 @@
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Thu, 18 Jul 2024 17:51:27 GMT"
"Wed, 24 Jul 2024 17:37:25 GMT"
],
"expires": [
"Thu, 18 Jul 2024 17:51:27 GMT"
"Wed, 24 Jul 2024 17:37:25 GMT"
],
"origin-trial": [
"AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@ -62,8 +62,8 @@
"ESF"
],
"set-cookie": [
"YSC\u003dCE3yzBeFLQI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dFri, 22-Oct-2021 17:51:27 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
"YSC\u003dQqImeZ_ECz4; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dThu, 28-Oct-2021 17:37:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"

12
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/shorts/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

12
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/shorts/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

3500
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/shorts/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long