[YouTube] Support pageHeader on user channels
Also move duplicate strings into constants and add a missing default switch case.
This commit is contained in:
AudricV
parent
df26badd4a
commit
8be64574e4
|
|
@ -23,7 +23,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
|
|||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
|
|
@ -59,6 +58,19 @@ import javax.annotation.Nullable;
|
|||
|
||||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
// Constants of objects used multiples from channel responses
|
||||
private static final String IMAGE = "image";
|
||||
private static final String CONTENTS = "contents";
|
||||
private static final String CONTENT_PREVIEW_IMAGE_VIEW_MODEL = "contentPreviewImageViewModel";
|
||||
private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel";
|
||||
private static final String TAB_RENDERER = "tabRenderer";
|
||||
private static final String CONTENT = "content";
|
||||
private static final String METADATA = "metadata";
|
||||
private static final String AVATAR = "avatar";
|
||||
private static final String THUMBNAILS = "thumbnails";
|
||||
private static final String SOURCES = "sources";
|
||||
private static final String BANNER = "banner";
|
||||
|
||||
private JsonObject jsonResponse;
|
||||
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
|
|
@ -95,28 +107,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
jsonResponse = data.jsonResponse;
|
||||
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
|
||||
channelId = data.channelId;
|
||||
channelAgeGateRenderer = getChannelAgeGateRenderer();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private JsonObject getChannelAgeGateRenderer() {
|
||||
return jsonResponse.getObject("contents")
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.flatMap(tab -> tab.getObject("tabRenderer")
|
||||
.getObject("content")
|
||||
.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast))
|
||||
.filter(content -> content.has("channelAgeGateRenderer"))
|
||||
.map(content -> content.getObject("channelAgeGateRenderer"))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -133,62 +124,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
assertPageFetched();
|
||||
return channelHeader.map(header -> header.json)
|
||||
.flatMap(header -> Optional.ofNullable(header.getString("channelId"))
|
||||
.or(() -> Optional.ofNullable(header.getObject("navigationEndpoint")
|
||||
.getObject("browseEndpoint")
|
||||
.getString("browseId"))
|
||||
))
|
||||
.or(() -> Optional.ofNullable(channelId))
|
||||
.orElseThrow(() -> new ParsingException("Could not get channel ID"));
|
||||
return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
assertPageFetched();
|
||||
if (channelAgeGateRenderer != null) {
|
||||
final String title = channelAgeGateRenderer.getString("channelTitle");
|
||||
if (isNullOrEmpty(title)) {
|
||||
throw new ParsingException("Could not get channel name");
|
||||
}
|
||||
return title;
|
||||
}
|
||||
|
||||
final String metadataRendererTitle = jsonResponse.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("title");
|
||||
if (!isNullOrEmpty(metadataRendererTitle)) {
|
||||
return metadataRendererTitle;
|
||||
}
|
||||
|
||||
return channelHeader.map(header -> {
|
||||
final JsonObject channelJson = header.json;
|
||||
switch (header.headerType) {
|
||||
case PAGE:
|
||||
return channelJson.getObject("content")
|
||||
.getObject("pageHeaderViewModel")
|
||||
.getObject("title")
|
||||
.getObject("dynamicTextViewModel")
|
||||
.getObject("text")
|
||||
.getString("content", channelJson.getString("pageTitle"));
|
||||
|
||||
case CAROUSEL:
|
||||
case INTERACTIVE_TABBED:
|
||||
return getTextFromObject(channelJson.getObject("title"));
|
||||
|
||||
case C4_TABBED:
|
||||
default:
|
||||
return channelJson.getString("title");
|
||||
}
|
||||
})
|
||||
// The channel name from a microformatDataRenderer may be different from the one displayed,
|
||||
// especially for auto-generated channels, depending on the language requested for the
|
||||
// interface (hl parameter of InnerTube requests' payload)
|
||||
.or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
|
||||
.getObject("microformatDataRenderer")
|
||||
.getString("title")))
|
||||
.orElseThrow(() -> new ParsingException("Could not get channel name"));
|
||||
return YoutubeChannelHelper.getChannelName(
|
||||
channelHeader, jsonResponse, channelAgeGateRenderer);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -196,8 +140,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
public List<Image> getAvatars() throws ParsingException {
|
||||
assertPageFetched();
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
|
||||
.getArray("thumbnails"))
|
||||
return Optional.ofNullable(channelAgeGateRenderer.getObject(AVATAR)
|
||||
.getArray(THUMBNAILS))
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatars"));
|
||||
}
|
||||
|
|
@ -205,22 +149,35 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
return channelHeader.map(header -> {
|
||||
switch (header.headerType) {
|
||||
case PAGE:
|
||||
return header.json.getObject("content")
|
||||
.getObject("pageHeaderViewModel")
|
||||
.getObject("image")
|
||||
.getObject("contentPreviewImageViewModel")
|
||||
.getObject("image")
|
||||
.getArray("sources");
|
||||
final JsonObject imageObj = header.json.getObject(CONTENT)
|
||||
.getObject(PAGE_HEADER_VIEW_MODEL)
|
||||
.getObject(IMAGE);
|
||||
|
||||
if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
|
||||
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
|
||||
.getObject(IMAGE)
|
||||
.getArray(SOURCES);
|
||||
}
|
||||
|
||||
if (imageObj.has("decoratedAvatarViewModel")) {
|
||||
return imageObj.getObject("decoratedAvatarViewModel")
|
||||
.getObject(AVATAR)
|
||||
.getObject("avatarViewModel")
|
||||
.getObject(IMAGE)
|
||||
.getArray(SOURCES);
|
||||
}
|
||||
|
||||
// Return an empty avatar array as a fallback
|
||||
return new JsonArray();
|
||||
case INTERACTIVE_TABBED:
|
||||
return header.json.getObject("boxArt")
|
||||
.getArray("thumbnails");
|
||||
.getArray(THUMBNAILS);
|
||||
|
||||
case C4_TABBED:
|
||||
case CAROUSEL:
|
||||
default:
|
||||
return header.json.getObject("avatar")
|
||||
.getArray("thumbnails");
|
||||
return header.json.getObject(AVATAR)
|
||||
.getArray(THUMBNAILS);
|
||||
}
|
||||
})
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
|
|
@ -235,10 +192,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
return List.of();
|
||||
}
|
||||
|
||||
// No banner is available on pageHeaderRenderer headers
|
||||
return channelHeader.filter(header -> header.headerType != HeaderType.PAGE)
|
||||
.map(header -> header.json.getObject("banner")
|
||||
.getArray("thumbnails"))
|
||||
return channelHeader.map(header -> {
|
||||
if (header.headerType == HeaderType.PAGE) {
|
||||
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
|
||||
.getObject(PAGE_HEADER_VIEW_MODEL);
|
||||
|
||||
if (pageHeaderViewModel.has(BANNER)) {
|
||||
return pageHeaderViewModel.getObject(BANNER)
|
||||
.getObject("imageBannerViewModel")
|
||||
.getObject(IMAGE)
|
||||
.getArray(SOURCES);
|
||||
}
|
||||
|
||||
// No banner is available (this should happen on pageHeaderRenderers of
|
||||
// system channels), use an empty JsonArray instead
|
||||
return new JsonArray();
|
||||
}
|
||||
|
||||
return header.json
|
||||
.getObject(BANNER)
|
||||
.getArray(THUMBNAILS);
|
||||
})
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
.orElse(List.of());
|
||||
}
|
||||
|
|
@ -264,14 +238,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
if (channelHeader.isPresent()) {
|
||||
final ChannelHeader header = channelHeader.get();
|
||||
|
||||
if (header.headerType == HeaderType.INTERACTIVE_TABBED
|
||||
|| header.headerType == HeaderType.PAGE) {
|
||||
// No subscriber count is available on interactiveTabbedHeaderRenderer and
|
||||
// pageHeaderRenderer headers
|
||||
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
|
||||
// No subscriber count is available on interactiveTabbedHeaderRenderer header
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
final JsonObject headerJson = header.json;
|
||||
if (header.headerType == HeaderType.PAGE) {
|
||||
return getSubscriberCountFromPageChannelHeader(headerJson);
|
||||
}
|
||||
|
||||
JsonObject textObject = null;
|
||||
|
||||
if (headerJson.has("subscriberCountText")) {
|
||||
|
|
@ -292,6 +268,51 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
private long getSubscriberCountFromPageChannelHeader(@Nonnull final JsonObject headerJson)
|
||||
throws ParsingException {
|
||||
final JsonObject metadataObject = headerJson.getObject(CONTENT)
|
||||
.getObject(PAGE_HEADER_VIEW_MODEL)
|
||||
.getObject(METADATA);
|
||||
if (metadataObject.has("contentMetadataViewModel")) {
|
||||
final JsonArray metadataPart = metadataObject.getObject("contentMetadataViewModel")
|
||||
.getArray("metadataRows")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(metadataRow -> metadataRow.getArray("metadataParts"))
|
||||
/*
|
||||
Find metadata parts which have two elements: channel handle and subscriber
|
||||
count.
|
||||
|
||||
On autogenerated music channels, the subscriber count is not shown with this
|
||||
header.
|
||||
|
||||
Use the first metadata parts object found.
|
||||
*/
|
||||
.filter(metadataParts -> metadataParts.size() == 2)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (metadataPart == null) {
|
||||
// As the parsing of the metadata parts object needed to get the subscriber count
|
||||
// is fragile, return UNKNOWN_SUBSCRIBER_COUNT when it cannot be got
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
try {
|
||||
// The subscriber count is at the same position for all languages as of 02/03/2024
|
||||
return Utils.mixedNumberWordToLong(metadataPart.getObject(0)
|
||||
.getObject("text")
|
||||
.getString(CONTENT));
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
}
|
||||
|
||||
// If the channel header has no contentMetadataViewModel (which is the case for system
|
||||
// channels using this header), return UNKNOWN_SUBSCRIBER_COUNT
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
|
@ -302,12 +323,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
try {
|
||||
if (channelHeader.isPresent()) {
|
||||
final ChannelHeader header = channelHeader.get();
|
||||
|
||||
if (header.headerType == HeaderType.PAGE) {
|
||||
// A pageHeaderRenderer doesn't contain a description
|
||||
return null;
|
||||
}
|
||||
|
||||
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
|
||||
/*
|
||||
In an interactiveTabbedHeaderRenderer, the real description, is only available
|
||||
|
|
@ -322,7 +337,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
|
||||
// The description is cut and the original one can be only accessed from the About tab
|
||||
return jsonResponse.getObject("metadata")
|
||||
return jsonResponse.getObject("title")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("description");
|
||||
} catch (final Exception e) {
|
||||
|
|
@ -371,7 +386,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Nonnull
|
||||
private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
|
||||
final JsonArray responseTabs = jsonResponse.getObject("contents")
|
||||
final JsonArray responseTabs = jsonResponse.getObject(CONTENTS)
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs");
|
||||
|
||||
|
|
@ -392,8 +407,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
responseTabs.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(tab -> tab.has("tabRenderer"))
|
||||
.map(tab -> tab.getObject("tabRenderer"))
|
||||
.filter(tab -> tab.has(TAB_RENDERER))
|
||||
.map(tab -> tab.getObject(TAB_RENDERER))
|
||||
.forEach(tabRenderer -> {
|
||||
final String tabUrl = tabRenderer.getObject("endpoint")
|
||||
.getObject("commandMetadata")
|
||||
|
|
@ -432,6 +447,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
case "playlists":
|
||||
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
|
||||
break;
|
||||
default:
|
||||
// Unsupported channel tab, ignore it
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper
|
|||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
|
|
@ -120,60 +119,13 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
final String id = jsonResponse.getObject("header")
|
||||
.getObject("c4TabbedHeaderRenderer")
|
||||
.getString("channelId", "");
|
||||
|
||||
if (!id.isEmpty()) {
|
||||
return id;
|
||||
return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
|
||||
}
|
||||
|
||||
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
|
||||
.getObject("carouselHeaderRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(item -> item.has("topicChannelDetailsRenderer"))
|
||||
.findFirst()
|
||||
.flatMap(item ->
|
||||
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
|
||||
.getObject("navigationEndpoint")
|
||||
.getObject("browseEndpoint")
|
||||
.getString("browseId")));
|
||||
if (carouselHeaderId.isPresent()) {
|
||||
return carouselHeaderId.get();
|
||||
}
|
||||
|
||||
if (!isNullOrEmpty(channelId)) {
|
||||
return channelId;
|
||||
} else {
|
||||
throw new ParsingException("Could not get channel ID");
|
||||
}
|
||||
}
|
||||
|
||||
protected String getChannelName() {
|
||||
final String metadataName = jsonResponse.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("title");
|
||||
if (!isNullOrEmpty(metadataName)) {
|
||||
return metadataName;
|
||||
}
|
||||
|
||||
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
|
||||
.map(header -> {
|
||||
final Object title = header.json.get("title");
|
||||
if (title instanceof String) {
|
||||
return (String) title;
|
||||
} else if (title instanceof JsonObject) {
|
||||
final String headerName = getTextFromObject((JsonObject) title);
|
||||
if (!isNullOrEmpty(headerName)) {
|
||||
return headerName;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
})
|
||||
.orElse("");
|
||||
protected String getChannelName() throws ParsingException {
|
||||
return YoutubeChannelHelper.getChannelName(
|
||||
channelHeader, jsonResponse,
|
||||
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
|
|||
Loading…
Reference in New Issue