be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1094 from AudricV/yt_support-more-channel-headers 

[YouTube] Support more channel headers
This commit is contained in:
Stypox 2023-08-12 11:08:30 +02:00 committed by GitHub
parent 7936987955 44b664af15
commit 93a90b816d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 875 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View File

@ -219,6 +219,50 @@ public final class YoutubeChannelHelper {
*/
public static final class ChannelHeader {
/**
* Types of supported YouTube channel headers.
*/
public enum HeaderType {
/**
* A {@code c4TabbedHeaderRenderer} channel header type.
*
* <p>
* This header is returned on the majority of channels and contains the channel's name,
* its banner and its avatar and its subscriber count in most cases.
* </p>
*/
C4_TABBED,
/**
* An {@code interactiveTabbedHeaderRenderer} channel header type.
*
* <p>
* This header is returned for gaming topic channels, and only contains the channel's
* name, its banner and a poster as its "avatar".
* </p>
*/
INTERACTIVE_TABBED,
/**
* A {@code carouselHeaderRenderer} channel header type.
*
* <p>
* This header returns only the channel's name, its avatar and its subscriber count.
* </p>
*/
CAROUSEL,
/**
* A {@code pageHeaderRenderer} channel header type.
*
* <p>
* This header returns only the channel's name and its avatar.
* </p>
*/
PAGE
}
/**
* The channel header JSON response.
*/
@ -226,17 +270,17 @@ public final class YoutubeChannelHelper {
public final JsonObject json;
/**
* Whether the header is a {@code carouselHeaderRenderer}.
* The type of the channel header.
*
* <p>
* See the class documentation for more details.
* See the documentation of the {@link HeaderType} class for more details.
* </p>
*/
public final boolean isCarouselHeader;
public final HeaderType headerType;
private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
this.headerType = headerType;
}
}
@ -254,7 +298,7 @@ public final class YoutubeChannelHelper {
if (header.has("c4TabbedHeaderRenderer")) {
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
} else if (header.has("carouselHeaderRenderer")) {
return header.getObject("carouselHeaderRenderer")
.getArray("contents")
@ -264,7 +308,14 @@ public final class YoutubeChannelHelper {
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.map(item -> item.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
} else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer"))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE));
} else if (header.has("interactiveTabbedHeaderRenderer")) {
return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json,
ChannelHeader.HeaderType.INTERACTIVE_TABBED));
} else {
return Optional.empty();
}

204
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -17,6 +17,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.ChannelHeader;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.ChannelHeader.HeaderType;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
@ -59,7 +61,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
private Optional<ChannelHeader> channelHeader;
private String channelId;
@ -116,11 +118,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.orElse(null);
}
@Nonnull
private Optional<JsonObject> getChannelHeaderJson() {
return channelHeader.map(it -> it.json);
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
@ -134,7 +131,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
return getChannelHeaderJson()
assertPageFetched();
return channelHeader.map(header -> header.json)
.flatMap(header -> Optional.ofNullable(header.getString("channelId"))
.or(() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.getObject("browseEndpoint")
@ -147,8 +145,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getName() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return channelAgeGateRenderer.getString("channelTitle");
final String title = channelAgeGateRenderer.getString("channelTitle");
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get channel name");
}
return title;
}
final String metadataRendererTitle = jsonResponse.getObject("metadata")
@ -158,53 +161,105 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return metadataRendererTitle;
}
return getChannelHeaderJson().flatMap(header -> {
final Object title = header.get("title");
if (title instanceof String) {
return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return Optional.of(headerName);
}
return channelHeader.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:
return channelJson.getObject("content")
.getObject("pageHeaderViewModel")
.getObject("title")
.getObject("dynamicTextViewModel")
.getObject("text")
.getString("content", channelJson.getString("pageTitle"));
case CAROUSEL:
case INTERACTIVE_TABBED:
return getTextFromObject(channelJson.getObject("title"));
case C4_TABBED:
default:
return channelJson.getString("title");
}
return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
})
// The channel name from a microformatDataRenderer may be different from the one displayed,
// especially for auto-generated channels, depending on the language requested for the
// interface (hl parameter of InnerTube requests' payload)
.or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getString("title")))
.orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Override
public String getAvatarUrl() throws ParsingException {
final JsonObject avatarJsonObjectContainer;
assertPageFetched();
if (channelAgeGateRenderer != null) {
avatarJsonObjectContainer = channelAgeGateRenderer;
} else {
avatarJsonObjectContainer = getChannelHeaderJson()
return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
}
return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url"));
return channelHeader.map(header -> {
switch (header.headerType) {
case PAGE:
return header.json.getObject("content")
.getObject("pageHeaderViewModel")
.getObject("image")
.getObject("contentPreviewImageViewModel")
.getObject("image")
.getArray("sources")
.getObject(0)
.getString("url");
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray("thumbnails")
.getObject(0)
.getString("url");
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url");
}
})
.map(YoutubeParsingHelper::fixThumbnailUrl)
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
}
@Override
public String getBannerUrl() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return "";
return null;
}
return getChannelHeaderJson().flatMap(header -> Optional.ofNullable(
header.getObject("banner")
.getArray("thumbnails")
.getObject(0)
.getString("url")))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no banner is
// found
// Return null in this case
.orElse(null);
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.PAGE) {
// No banner is available on pageHeaderRenderer headers
return null;
}
return Optional.ofNullable(header.json.getObject("banner")
.getArray("thumbnails")
.getObject(0)
.getString("url"))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no
// banner is found
// Return null in this case
.orElse(null);
}
return null;
}
@Override
@ -214,25 +269,34 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (final Exception e) {
throw new ParsingException("Could not get feed url", e);
throw new ParsingException("Could not get feed URL", e);
}
}
@Override
public long getSubscriberCount() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return UNKNOWN_SUBSCRIBER_COUNT;
}
final Optional<JsonObject> headerOpt = getChannelHeaderJson();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get();
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED
|| header.headerType == HeaderType.PAGE) {
// No subscriber count is available on interactiveTabbedHeaderRenderer and
// pageHeaderRenderer headers
return UNKNOWN_SUBSCRIBER_COUNT;
}
final JsonObject headerJson = header.json;
JsonObject textObject = null;
if (header.has("subscriberCountText")) {
textObject = header.getObject("subscriberCountText");
} else if (header.has("subtitle")) {
textObject = header.getObject("subtitle");
if (headerJson.has("subscriberCountText")) {
textObject = headerJson.getObject("subscriberCountText");
} else if (headerJson.has("subtitle")) {
textObject = headerJson.getObject("subtitle");
}
if (textObject != null) {
@ -249,11 +313,34 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getDescription() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return null;
}
try {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.PAGE) {
// A pageHeaderRenderer doesn't contain a description
return null;
}
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(header.json.getObject("description"));
}
}
// The description is cut and the original one can be only accessed from the About tab
return jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("description");
@ -279,27 +366,39 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isVerified() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return false;
}
if (channelHeader.isPresent()) {
final YoutubeChannelHelper.ChannelHeader header = channelHeader.get();
final ChannelHeader header = channelHeader.get();
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
// carouselHeaderRenderer and pageHeaderRenderer does not contain any verification
// badges
// Since they are only shown on YouTube internal channels or on channels of large
// organizations broadcasting live events, we can assume the channel to be verified
if (header.headerType == HeaderType.CAROUSEL || header.headerType == HeaderType.PAGE) {
return true;
}
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
// If the header has an autoGenerated property, it should mean that the channel has
// been auto generated by YouTube: we can assume the channel to be verified in this
// case
return header.json.has("autoGenerated");
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
}
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer == null) {
return getTabsForNonAgeRestrictedChannels();
}
@ -401,6 +500,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
assertPageFetched();
if (channelAgeGateRenderer != null) {
return List.of();
}

108
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
@ -710,8 +711,10 @@ public class YoutubeChannelExtractorTest {
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testDescription() throws ParsingException {
assertNotBlank(extractor.getDescription());
}
@Test
@ -885,4 +888,109 @@ public class YoutubeChannelExtractorTest {
assertTrue(extractor.getTags().isEmpty());
}
}
static class InteractiveTabbedHeader implements BaseChannelExtractorTest {
private static ChannelExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "interactiveTabbedHeader"));
extractor = YouTube.getChannelExtractor(
"https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg");
extractor.fetchPage();
}
@Test
@Override
public void testDescription() throws Exception {
final String description = extractor.getDescription();
assertContains("Minecraft", description);
assertContains("game", description);
assertContains("Mojang", description);
}
@Test
@Override
public void testAvatarUrl() throws Exception {
final String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
assertContains("yt3", avatarUrl);
}
@Test
@Override
public void testBannerUrl() throws Exception {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertContains("yt3", bannerUrl);
}
@Test
@Override
public void testFeedUrl() throws Exception {
assertEquals(
"https://www.youtube.com/feeds/videos.xml?channel_id=UCQvWX73GQygcwXOTSf_VDVg",
extractor.getFeedUrl());
}
@Test
@Override
public void testSubscriberCount() throws Exception {
// Subscriber count is not available on channels with an interactiveTabbedHeaderRenderer
assertEquals(ChannelExtractor.UNKNOWN_SUBSCRIBER_COUNT, extractor.getSubscriberCount());
}
@Test
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
// Gaming topic channels tabs are not yet supported, so an empty list should be returned
assertTrue(extractor.getTabs().isEmpty());
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
@Test
@Override
public void testServiceId() throws Exception {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertContains("Minecraft", extractor.getName());
}
@Test
@Override
public void testId() throws Exception {
assertEquals("UCQvWX73GQygcwXOTSf_VDVg", extractor.getId());
}
@Test
@Override
public void testUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg",
extractor.getOriginalUrl());
}
}
}

85
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/interactiveTabbedHeader/generated_mock_0.json Normal file
View File

@ -0,0 +1,85 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Referer": [
"https://www.youtube.com"
],
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 08 Aug 2023 17:04:33 GMT"
],
"expires": [
"Tue, 08 Aug 2023 17:04:33 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dV1ZtomDP-24; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 11-Nov-2020 17:04:33 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+547; expires\u003dThu, 07-Aug-2025 17:04:33 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

79
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/interactiveTabbedHeader/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

393
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/interactiveTabbedHeader/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long