be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #920 from AudricV/fix-yt-trending-extraction 

[YouTube] Fix extraction of the Trending kiosk with the new data model returned
This commit is contained in:
Stypox 2022-09-14 11:14:13 +02:00 committed by GitHub
parent 884da40f65 d8ddeb549c
commit 14ef430546
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 62 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -1,26 +1,25 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
/* /*
* Created by Christian Schabesberger on 12.08.17. * Created by Christian Schabesberger on 12.08.17.
* *
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* YoutubeTrendingExtractor.java is part of NewPipe. * YoutubeTrendingExtractor.java is part of NewPipe Extractor.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version. * (at your option) any later version.
* *
* NewPipe is distributed in the hope that it will be useful, * NewPipe Extractor is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details. * GNU General Public License for more details.
* *
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/ */
import com.grack.nanojson.JsonArray; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
@ -92,25 +91,66 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
final JsonArray itemSectionRenderers = initialData.getObject("contents") final JsonObject tabContent = getTrendingTabContent();
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents");
for (final Object itemSectionRenderer : itemSectionRenderers) { if (tabContent.has("richGridRenderer")) {
final JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer) tabContent.getObject("richGridRenderer")
.getObject("itemSectionRenderer").getArray("contents").getObject(0) .getArray("contents")
.getObject("shelfRenderer").getObject("content") .stream()
.getObject("expandedShelfContentsRenderer"); .filter(JsonObject.class::isInstance)
for (final Object ul : expandedShelfContentsRenderer.getArray("items")) { .map(JsonObject.class::cast)
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); // Filter Trending shorts and Recently trending sections
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); .filter(content -> content.has("richItemRenderer"))
} .map(content -> content.getObject("richItemRenderer")
.getObject("content")
.getObject("videoRenderer"))
.forEachOrdered(videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
} else if (tabContent.has("sectionListRenderer")) {
tabContent.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(content -> content.getObject("itemSectionRenderer")
.getArray("contents")
.stream())
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(content -> content.getObject("shelfRenderer"))
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
.filter(shelfRenderer -> !shelfRenderer.has("title"))
.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.getObject("expandedShelfContentsRenderer")
.getArray("items")
.stream())
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(item -> item.getObject("videoRenderer"))
.forEachOrdered(videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
} }
return new InfoItemsPage<>(collector, null); return new InfoItemsPage<>(collector, null);
} }
private JsonObject getTrendingTabContent() throws ParsingException {
return initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(tab -> tab.getObject("tabRenderer"))
.filter(tabRenderer -> tabRenderer.getBoolean("selected"))
.filter(tabRenderer -> tabRenderer.has("content"))
// There should be at most one tab selected
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
.getObject("content");
}
} }