be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1045 from Theta-Dev/fix/trending-video-tab 

[YouTube] Extract trends from A/B tested "Videos" tab and fix extraction of trends name from A/B tested new title design
This commit is contained in:
Audric V 2023-05-19 11:22:49 +02:00 committed by GitHub
parent 92a0024424 3673d4ae01
commit 533121fb81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 168 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -41,12 +41,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData;
private static final String VIDEOS_TAB_PARAMS = "4gIOGgxtb3N0X3BvcHVsYXI%3D";
public YoutubeTrendingExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final String kioskId) {
@ -60,6 +63,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("browseId", "FEtrending")
.value("params", VIDEOS_TAB_PARAMS)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
@ -81,6 +85,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
} else if (header.has("c4TabbedHeaderRenderer")) {
name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
} else if (header.has("pageHeaderRenderer")) {
name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle");
}
if (isNullOrEmpty(name)) {
@ -94,7 +100,10 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
final JsonObject tabContent = getTrendingTabContent();
final JsonObject tab = getTrendingTab();
final JsonObject tabContent = tab.getObject("content");
final boolean isVideoTab = tab.getObject("endpoint").getObject("browseEndpoint")
.getString("params", "").equals(VIDEOS_TAB_PARAMS);
if (tabContent.has("richGridRenderer")) {
tabContent.getObject("richGridRenderer")
@ -110,7 +119,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.forEachOrdered(videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
} else if (tabContent.has("sectionListRenderer")) {
tabContent.getObject("sectionListRenderer")
final Stream<JsonObject> shelves = tabContent.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
@ -120,11 +129,19 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.stream())
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(content -> content.getObject("shelfRenderer"))
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
.filter(shelfRenderer -> !shelfRenderer.has("title"))
.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.map(content -> content.getObject("shelfRenderer"));
final Stream<JsonObject> items;
if (isVideoTab) {
// The first shelf of the Videos tab contains the normal trends
items = shelves.findFirst().stream();
} else {
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title"));
}
items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.getObject("expandedShelfContentsRenderer")
.getArray("items")
.stream())
@ -138,7 +155,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
return new InfoItemsPage<>(collector, null);
}
private JsonObject getTrendingTabContent() throws ParsingException {
private JsonObject getTrendingTab() throws ParsingException {
return initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
@ -150,7 +167,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
.filter(tabRenderer -> tabRenderer.has("content"))
// There should be at most one tab selected
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
.getObject("content");
.orElseThrow(() ->
new ParsingException("Could not get \"Now\" or \"Videos\" trending tab"));
}
}

21
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json
View File

@ -3,10 +3,10 @@
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"Referer": [
"https://www.youtube.com"
],
"Referer": [
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
@ -29,7 +29,7 @@
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
@ -37,14 +37,17 @@
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 22 Nov 2022 10:40:53 GMT"
"Sun, 16 Apr 2023 17:42:25 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:40:53 GMT"
"Sun, 16 Apr 2023 17:42:25 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@ -59,9 +62,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003dKWKE6LaMJTE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:53 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+649; expires\u003dThu, 21-Nov-2024 10:40:53 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003deNq1o5_KNzg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:42:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+549; expires\u003dTue, 15-Apr-2025 17:42:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"

17
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

60
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

21
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json
View File

@ -3,10 +3,10 @@
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"Referer": [
"https://www.youtube.com"
],
"Referer": [
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
@ -29,7 +29,7 @@
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
@ -37,14 +37,17 @@
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 22 Nov 2022 10:40:26 GMT"
"Sun, 16 Apr 2023 17:35:48 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:40:26 GMT"
"Sun, 16 Apr 2023 17:35:48 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@ -59,9 +62,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003daSSq4mC6HTI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+953; expires\u003dThu, 21-Nov-2024 10:40:26 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003dKx2AfujDxNk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:35:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+605; expires\u003dTue, 15-Apr-2025 17:35:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"

15
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

60
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long