be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1045 from Theta-Dev/fix/trending-video-tab 

[YouTube] Extract trends from A/B tested "Videos" tab and fix extraction of trends name from A/B tested new title design
This commit is contained in:
Audric V 2023-05-19 11:22:49 +02:00 committed by GitHub
parent 92a0024424 3673d4ae01
commit 533121fb81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 168 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -41,12 +41,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.stream.Stream;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> { public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData; private JsonObject initialData;
private static final String VIDEOS_TAB_PARAMS = "4gIOGgxtb3N0X3BvcHVsYXI%3D";
public YoutubeTrendingExtractor(final StreamingService service, public YoutubeTrendingExtractor(final StreamingService service,
final ListLinkHandler linkHandler, final ListLinkHandler linkHandler,
final String kioskId) { final String kioskId) {
@ -60,6 +63,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry()) getExtractorContentCountry())
.value("browseId", "FEtrending") .value("browseId", "FEtrending")
.value("params", VIDEOS_TAB_PARAMS)
.done()) .done())
.getBytes(StandardCharsets.UTF_8); .getBytes(StandardCharsets.UTF_8);
// @formatter:on // @formatter:on
@ -81,6 +85,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title"); name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
} else if (header.has("c4TabbedHeaderRenderer")) { } else if (header.has("c4TabbedHeaderRenderer")) {
name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title"); name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
} else if (header.has("pageHeaderRenderer")) {
name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle");
} }
if (isNullOrEmpty(name)) { if (isNullOrEmpty(name)) {
@ -94,7 +100,10 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
final JsonObject tabContent = getTrendingTabContent(); final JsonObject tab = getTrendingTab();
final JsonObject tabContent = tab.getObject("content");
final boolean isVideoTab = tab.getObject("endpoint").getObject("browseEndpoint")
.getString("params", "").equals(VIDEOS_TAB_PARAMS);
if (tabContent.has("richGridRenderer")) { if (tabContent.has("richGridRenderer")) {
tabContent.getObject("richGridRenderer") tabContent.getObject("richGridRenderer")
@ -110,7 +119,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.forEachOrdered(videoRenderer -> collector.commit( .forEachOrdered(videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
} else if (tabContent.has("sectionListRenderer")) { } else if (tabContent.has("sectionListRenderer")) {
tabContent.getObject("sectionListRenderer") final Stream<JsonObject> shelves = tabContent.getObject("sectionListRenderer")
.getArray("contents") .getArray("contents")
.stream() .stream()
.filter(JsonObject.class::isInstance) .filter(JsonObject.class::isInstance)
@ -120,11 +129,19 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.stream()) .stream())
.filter(JsonObject.class::isInstance) .filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast) .map(JsonObject.class::cast)
.map(content -> content.getObject("shelfRenderer")) .map(content -> content.getObject("shelfRenderer"));
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends final Stream<JsonObject> items;
.filter(shelfRenderer -> !shelfRenderer.has("title")) if (isVideoTab) {
.flatMap(shelfRenderer -> shelfRenderer.getObject("content") // The first shelf of the Videos tab contains the normal trends
items = shelves.findFirst().stream();
} else {
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title"));
}
items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.getObject("expandedShelfContentsRenderer") .getObject("expandedShelfContentsRenderer")
.getArray("items") .getArray("items")
.stream()) .stream())
@ -138,7 +155,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
return new InfoItemsPage<>(collector, null); return new InfoItemsPage<>(collector, null);
} }
private JsonObject getTrendingTabContent() throws ParsingException { private JsonObject getTrendingTab() throws ParsingException {
return initialData.getObject("contents") return initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer") .getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs") .getArray("tabs")
@ -150,7 +167,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.filter(tabRenderer -> tabRenderer.has("content")) .filter(tabRenderer -> tabRenderer.has("content"))
// There should be at most one tab selected // There should be at most one tab selected
.findFirst() .findFirst()
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab")) .orElseThrow(() ->
.getObject("content"); new ParsingException("Could not get \"Now\" or \"Videos\" trending tab"));
} }
} }

21
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json
View File

@ -3,10 +3,10 @@
"httpMethod": "GET", "httpMethod": "GET",
"url": "https://www.youtube.com/sw.js", "url": "https://www.youtube.com/sw.js",
"headers": { "headers": {
"Origin": [ "Referer": [
"https://www.youtube.com" "https://www.youtube.com"
], ],
"Referer": [ "Origin": [
"https://www.youtube.com" "https://www.youtube.com"
], ],
"Accept-Language": [ "Accept-Language": [
@ -29,7 +29,7 @@
"https://www.youtube.com" "https://www.youtube.com"
], ],
"alt-svc": [ "alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\"" "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
], ],
"cache-control": [ "cache-control": [
"private, max-age\u003d0" "private, max-age\u003d0"
@ -37,14 +37,17 @@
"content-type": [ "content-type": [
"text/javascript; charset\u003dutf-8" "text/javascript; charset\u003dutf-8"
], ],
"cross-origin-opener-policy-report-only": [ "cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\"" "same-origin; report-to\u003d\"youtube_main\""
], ],
"date": [ "date": [
"Tue, 22 Nov 2022 10:40:53 GMT" "Sun, 16 Apr 2023 17:42:25 GMT"
], ],
"expires": [ "expires": [
"Tue, 22 Nov 2022 10:40:53 GMT" "Sun, 16 Apr 2023 17:42:25 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
], ],
"p3p": [ "p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\"" "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@ -59,9 +62,9 @@
"ESF" "ESF"
], ],
"set-cookie": [ "set-cookie": [
"YSC\u003dKWKE6LaMJTE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone", "YSC\u003deNq1o5_KNzg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:53 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone", "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:42:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+649; expires\u003dThu, 21-Nov-2024 10:40:53 GMT; path\u003d/; domain\u003d.youtube.com; Secure" "CONSENT\u003dPENDING+549; expires\u003dTue, 15-Apr-2025 17:42:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
], ],
"strict-transport-security": [ "strict-transport-security": [
"max-age\u003d31536000" "max-age\u003d31536000"

17
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

60
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

21
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json
View File

@ -3,10 +3,10 @@
"httpMethod": "GET", "httpMethod": "GET",
"url": "https://www.youtube.com/sw.js", "url": "https://www.youtube.com/sw.js",
"headers": { "headers": {
"Origin": [ "Referer": [
"https://www.youtube.com" "https://www.youtube.com"
], ],
"Referer": [ "Origin": [
"https://www.youtube.com" "https://www.youtube.com"
], ],
"Accept-Language": [ "Accept-Language": [
@ -29,7 +29,7 @@
"https://www.youtube.com" "https://www.youtube.com"
], ],
"alt-svc": [ "alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\"" "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
], ],
"cache-control": [ "cache-control": [
"private, max-age\u003d0" "private, max-age\u003d0"
@ -37,14 +37,17 @@
"content-type": [ "content-type": [
"text/javascript; charset\u003dutf-8" "text/javascript; charset\u003dutf-8"
], ],
"cross-origin-opener-policy-report-only": [ "cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\"" "same-origin; report-to\u003d\"youtube_main\""
], ],
"date": [ "date": [
"Tue, 22 Nov 2022 10:40:26 GMT" "Sun, 16 Apr 2023 17:35:48 GMT"
], ],
"expires": [ "expires": [
"Tue, 22 Nov 2022 10:40:26 GMT" "Sun, 16 Apr 2023 17:35:48 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
], ],
"p3p": [ "p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\"" "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@ -59,9 +62,9 @@
"ESF" "ESF"
], ],
"set-cookie": [ "set-cookie": [
"YSC\u003daSSq4mC6HTI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone", "YSC\u003dKx2AfujDxNk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone", "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:35:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+953; expires\u003dThu, 21-Nov-2024 10:40:26 GMT; path\u003d/; domain\u003d.youtube.com; Secure" "CONSENT\u003dPENDING+605; expires\u003dTue, 15-Apr-2025 17:35:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
], ],
"strict-transport-security": [ "strict-transport-security": [
"max-age\u003d31536000" "max-age\u003d31536000"

15
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

60
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long