be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Do some improvements to YoutubeStreamExtractor 

Get the real name of the uploader (for autogenerated channels and music artist channels), like before the migration to the JSON pbj.
Do some other improvements, especially reformatting some code to be in the 100 characters line limit and use final where possible.
This commit is contained in:
TiA4f8R 2021-04-15 18:58:59 +02:00
parent 58ce9b04a1
commit b49ae547a3
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
6 changed files with 295 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -63,7 +63,7 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
private static final String HARDCODED_CLIENT_VERSION = "2.20210408.08.00";
private static final String HARDCODED_CLIENT_VERSION = "2.20210413.07.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static String clientVersion;
private static String key;
@ -729,7 +729,7 @@ public class YoutubeParsingHelper {
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonBuilder<JsonObject> prepareJsonBuilder()
public static JsonBuilder<JsonObject> prepareJsonBuilder(final String contentCountry)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
@ -737,6 +737,8 @@ public class YoutubeParsingHelper {
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.value("hl", "en-GB")
.value("gl", contentCountry)
.end()
.end();
// @formatter:on

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -73,20 +73,23 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/
private String redirectedChannelId;
public YoutubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String channel_path = super.getId();
final String[] channelInfo = channel_path.split("/");
final String contentCountry = getExtractorContentCountry().getCountryCode();
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we
// couldn't get information about the channel associated with this URL, if there is one.
if (!channelInfo[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("url", "https://www.youtube.com/" + channel_path)
.done())
.getBytes(UTF_8);
@ -131,7 +134,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // equals to videos
.done())
@ -336,7 +339,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -361,7 +365,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
private Page getNextPageFrom(final JsonObject continuations) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
@ -370,7 +375,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);

69
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -40,7 +40,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonArray initialAjaxJson;
private JsonObject initialData;
private JsonObject playlistInfo;
@ -49,8 +48,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final byte[] body = JsonWriter.string(prepareJsonBuilder()
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // show unavailable videos
.done())
@ -63,15 +64,18 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private JsonObject getUploaderInfo() throws ParsingException {
final JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
final JsonArray items = initialData.getObject("sidebar")
.getObject("playlistSidebarRenderer").getArray("items");
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
JsonObject videoOwner = items.getObject(1)
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
// we might want to create a loop here instead of using duplicated code
videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
videoOwner = items.getObject(items.size())
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
@ -80,9 +84,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject getPlaylistInfo() throws ParsingException {
try {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items")
.getObject(0).getObject("playlistSidebarPrimaryInfoRenderer");
} catch (Exception e) {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer")
.getArray("items").getObject(0)
.getObject("playlistSidebarPrimaryInfoRenderer");
} catch (final Exception e) {
throw new ParsingException("Could not get PlaylistInfo", e);
}
}
@ -122,7 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderUrl() throws ParsingException {
try {
return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader url", e);
}
}
@ -131,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader name", e);
}
}
@ -142,7 +147,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e);
}
}
@ -157,7 +162,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get video count from playlist", e);
}
}
@ -186,18 +191,21 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
final JsonArray contents = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer")
.has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment)
.getObject("playlistSegmentRenderer").getObject("videoList")
.getObject("playlistVideoListRenderer").getArray("contents"));
}
}
@ -214,7 +222,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -235,7 +244,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
private Page getNextPageFrom(final JsonArray contents) throws IOException,
ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
@ -248,7 +258,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);
@ -260,12 +271,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object video : videos) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video)
.getObject("playlistVideoRenderer"), timeAgoParser) {
@Override
public long getViewCount() {
return -1;
@ -294,11 +307,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() {
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
return "";
/*final JsonArray thumbnails = initialAjaxJson.getObject(1)
.getObject("playerResponse")
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
return fixThumbnailUrl(url);*/
}
@Override

95
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -50,13 +50,16 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchExtractor extends SearchExtractor {
private JsonObject initialData;
public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
public YoutubeSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String query = super.getSearchString();
final String contentCountry = getExtractorContentCountry().getCountryCode();
// Get the search parameter of the request
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
@ -70,13 +73,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final byte[] body;
if (!isNullOrEmpty(params)) {
body = JsonWriter.string(prepareJsonBuilder()
body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("query", query)
.value("params", params)
.done())
.getBytes(UTF_8);
} else {
body = JsonWriter.string(prepareJsonBuilder()
body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("query", query)
.done())
.getBytes(UTF_8);
@ -100,11 +103,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getObject("itemSectionRenderer");
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) {
return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query");
return JsonUtils.getString(didYouMeanRenderer,
"correctedQueryEndpoint.searchEndpoint.query");
} else if (showingResultsForRenderer != null) {
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
} else {
@ -126,7 +131,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"));
.getObject("primaryContents").getObject("sectionListRenderer")
.getArray("contents"));
}
@Nonnull
@ -134,20 +140,23 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
final JsonArray sections = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents");
Page nextPage = null;
for (final Object section : sections) {
if (((JsonObject) section).has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
final JsonObject itemSectionRenderer = ((JsonObject) section)
.getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
} else if (((JsonObject) section).has("continuationItemRenderer")) {
nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer"));
nextPage = getNewNextPageFrom(((JsonObject) section)
.getObject("continuationItemRenderer"));
}
}
@ -155,7 +164,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -174,29 +184,15 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
} else {
// @formatter:off
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.value("utcOffsetMinutes", 0)
.end()
.object("request").end()
.object("user").end()
.end()
final byte[] json = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("continuation", page.getId())
.end().done().getBytes(UTF_8);
.done())
.getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList(this.getUrl()));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final String responseBody = getValidJsonResponseBody(getDownloader().post(
page.getUrl(), new HashMap<>(), json));
final JsonObject ajaxJson;
try {
@ -206,16 +202,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems");
.getObject(0).getObject("appendContinuationItemsAction")
.getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents");
final JsonArray contents = continuationItems.getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer")));
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1)
.getObject("continuationItemRenderer")));
}
}
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray contents) throws NothingFoundException, ParsingException {
private void collectStreamsFrom(final InfoItemsSearchCollector collector,
final JsonArray contents) throws NothingFoundException,
ParsingException {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object content : contents) {
@ -224,11 +225,14 @@ public class YoutubeSearchExtractor extends SearchExtractor {
throw new NothingFoundException(getTextFromObject(
item.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (item.has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(item.getObject("videoRenderer"), timeAgoParser));
collector.commit(new YoutubeStreamInfoItemExtractor(item
.getObject("videoRenderer"), timeAgoParser));
} else if (item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(item.getObject("channelRenderer")));
collector.commit(new YoutubeChannelInfoItemExtractor(item
.getObject("channelRenderer")));
} else if (item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(item.getObject("playlistRenderer")));
collector.commit(new YoutubePlaylistInfoItemExtractor(item
.getObject("playlistRenderer")));
}
}
}
@ -238,15 +242,18 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final JsonObject nextContinuationData = continuations.getObject(0)
.getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
final String clickTrackingParams = nextContinuationData
.getString("clickTrackingParams");
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation="
+ continuation + "&itct=" + clickTrackingParams);
}
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuationItemRenderer)) {
return null;
}

322
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,6 +4,11 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
@ -12,8 +17,15 @@ import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
@ -120,8 +132,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
final JsonObject micro =
playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
final JsonObject micro = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) {
return micro.getString("uploadDate");
} else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) {
@ -140,11 +152,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
Localization.fromLocalizationCode("en"));
OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (final Exception ignored) {
@ -159,8 +174,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
// TODO: this parses English formatted dates only, we need a better approach to parse
// the textual date
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer()
.getObject("dateText")),
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (final Exception ignored) {
@ -185,7 +202,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getThumbnailUrl() throws ParsingException {
assertPageFetched();
try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail")
.getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
@ -202,13 +220,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
// description with more info on links
try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
String description = getTextFromObject(getVideoSecondaryInfoRenderer()
.getObject("description"), true);
if (!isNullOrEmpty(description)) return new Description(description, Description.HTML);
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
String description = playerResponse.getObject("videoDetails").getString("shortDescription");
String description = playerResponse.getObject("videoDetails")
.getString("shortDescription");
if (description == null) {
final JsonObject descriptionObject = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer").getObject("description");
@ -289,22 +309,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() throws ParsingException {
assertPageFetched();
String views = null;
try {
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
if (isNullOrEmpty(views)) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count");
}
if (views.toLowerCase().contains("no views")) return 0;
final String views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count");
return Long.parseLong(Utils.removeNonDigitCharacters(views));
}
@ -320,13 +326,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (final NullPointerException e) {
// if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the like button is missing", e);
throw new ParsingException(
"Ratings are enabled even though the like button is missing", e);
}
return -1;
}
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
nfe);
} catch (final Exception e) {
if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get like count", e);
@ -338,6 +346,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getDislikeCount() throws ParsingException {
assertPageFetched();
String dislikesString = "";
try {
try {
@ -346,13 +355,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (final NullPointerException e) {
// if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
throw new ParsingException(
"Ratings are enabled even though the dislike button is missing", e);
}
return -1;
}
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer",
nfe);
} catch (final Exception e) {
if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get dislike count", e);
@ -366,16 +377,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException {
assertPageFetched();
try {
final String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (!isNullOrEmpty(uploaderUrl)) {
return uploaderUrl;
}
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
final String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (!isNullOrEmpty(uploaderId)) {
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
@ -389,19 +390,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderName() throws ParsingException {
assertPageFetched();
String uploaderName = null;
try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (final ParsingException ignored) {
}
if (isNullOrEmpty(uploaderName)) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name");
}
// Don't use the name in the videoSecondaryRenderer object to get real name of the uploader
// The difference between the real name of the channel and the displayed name is especially
// visible for music channels and autogenerated channels.
final String uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name");
return uploaderName;
}
@ -422,8 +415,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String url = null;
try {
url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
url = getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
@ -471,11 +465,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
if (!dashManifestUrl.contains("/signature/")) {
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)",
dashManifestUrl);
final String deobfuscatedSig;
deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig,
"/signature/" + deobfuscatedSig);
}
return dashManifestUrl;
@ -503,7 +499,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.AUDIO).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
@ -527,7 +524,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
ItagItem.ItagType.VIDEO).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
@ -551,7 +549,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
@ -620,27 +619,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM;
}
@Nullable
private StreamInfoItemExtractor getNextStream() throws ExtractionException {
try {
final JsonObject firstWatchNextItem = initialData.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
.getObject("secondaryResults").getArray("results").getObject(0);
if (!firstWatchNextItem.has("compactAutoplayRenderer")) {
// there is no "next" stream
return null;
}
final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer")
.getArray("contents").getObject(0).getObject("compactVideoRenderer");
return new YoutubeStreamInfoItemExtractor(videoInfo, getTimeAgoParser());
} catch (final Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedItems() throws ExtractionException {
@ -651,21 +629,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
try {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
getServiceId());
final StreamInfoItemExtractor nextStream = getNextStream();
if (nextStream != null) {
collector.commit(nextStream);
}
final JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("secondaryResults").getObject("secondaryResults").getArray("results");
final JsonArray results = initialData.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
.getObject("secondaryResults").getArray("results");
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object ul : results) {
if (((JsonObject) ul).has("compactVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser));
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul)
.getObject("compactVideoRenderer"), timeAgoParser));
}
}
return collector;
@ -680,9 +656,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getErrorMessage() {
try {
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse")
.getObject("playabilityStatus").getObject("errorScreen")
.getObject("playerErrorMessageRenderer").getObject("reason"));
return getTextFromObject(playerResponse.getObject("playabilityStatus")
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("reason"));
} catch (final ParsingException | NullPointerException e) {
return null; // no error message
}
@ -737,7 +713,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
youtubePlayerResponse = playerResponse;
}
JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse : playerResponse)
JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse
: playerResponse)
.getObject("playabilityStatus");
String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw the specific exception based on error message
@ -755,8 +732,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PrivateContentException("This video is private.");
}
} else if (reason.equals("Sign in to confirm your age")) {
// No streams can be fetched, therefore thrown an AgeRestrictedContentException explicitly.
throw new AgeRestrictedContentException("This age-restricted video cannot be watched.");
// No streams can be fetched, therefore thrown an AgeRestrictedContentException
// explicitly.
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched.");
}
}
if (status.equalsIgnoreCase("unplayable")) {
@ -772,7 +751,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PaidContentException("This video is only available for members of the channel of this video");
}
if (reason.equals("Video unavailable")) {
final String detailedErrorMessage = playabilityStatus.getObject("errorScreen")
final String detailedErrorMessage = playabilityStatus
.getObject("errorScreen")
.getObject("playerErrorMessageRenderer")
.getObject("subreason")
.getArray("runs")
@ -780,7 +760,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
.getString("text");
if (detailedErrorMessage != null) {
if (detailedErrorMessage.equals("The uploader has not made this video available in your country.")) {
throw new GeographicRestrictionException("This video is not available in user's country.");
throw new GeographicRestrictionException(
"This video is not available in user's country.");
}
}
}
@ -792,9 +773,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
final String videoInfoUrl = getVideoInfoUrl(getId());
final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization());
videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody()));
final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
final String infoPageResponse = NewPipe.getDownloader()
.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
try {
playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
@ -804,6 +787,39 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
try {
// Don't provide a video id to get a smaller response (around 9kb instead of 21 kb)
final String embedUrl = "https://www.youtube.com/embed/";
final String embedPageContent = NewPipe.getDownloader()
.get(embedUrl, getExtractorLocalization()).responseBody();
try {
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
} catch (final Parser.RegexException ex) {
// playerJsUrl is still available in the file, just somewhere else TODO
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (final Element elem : elems) {
if (elem.attr("src").contains("base.js")) {
playerJsUrl = elem.attr("src");
break;
}
}
}
// Get embed sts
return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent);
} catch (final Exception i) {
// if it fails we simply reply with no sts as then it does not seem to be necessary
return "";
}
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
@ -815,7 +831,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
}
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
throw new DeobfuscateException(
"Could not find deobfuscate function with any of the given patterns.", exception);
}
private String loadDeobfuscationCode()
@ -827,17 +844,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String functionPattern = "("
+ deobfuscationFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern,
playerCode) + ";";
final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction);
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(",
deobfuscateFunction);
final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
"(var " + helperObjectName.replace("$", "\\$")
+ "=\\{.+?\\}\\};)";
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
final String callerFunction =
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return "
+ deobfuscationFunctionName + "(a);}";
return helperObject + deobfuscateFunction + callerFunction;
} catch (final Exception e) {
@ -879,13 +900,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents");
final JsonArray contents = initialData.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoPrimaryInfoRenderer = null;
for (final Object content : contents) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer");
videoPrimaryInfoRenderer = ((JsonObject) content)
.getObject("videoPrimaryInfoRenderer");
break;
}
}
@ -901,13 +924,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents");
final JsonArray contents = initialData.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoSecondaryInfoRenderer = null;
for (final Object content : contents) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer");
videoSecondaryInfoRenderer = ((JsonObject) content)
.getObject("videoSecondaryInfoRenderer");
break;
}
}
@ -921,11 +946,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private static String getVideoInfoUrl(final String id) {
private static String getVideoInfoUrl(final String id, final String sts) {
// TODO: Try parsing embedded_player_response first
return "https://www.youtube.com/get_video_info?video_id=" + id +
"&eurl=https://youtube.googleapis.com/v/" + id +
"&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en";
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
private Map<String, ItagItem> getItags(final String streamingDataKey,
@ -969,15 +994,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject initRange = formatData.getObject("initRange");
JsonObject indexRange = formatData.getObject("indexRange");
String mimeType = formatData.getString("mimeType", EMPTY_STRING);
String codec = mimeType.contains("codecs") ? mimeType.split("\"")[1] : EMPTY_STRING;
String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
"-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
"-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
"-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
"-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
@ -1027,10 +1057,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
final String baseUrl = url.replace("$L", String.valueOf(i - 1))
.replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls;
if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
final int totalPages = (int) Math.ceil(totalCount / (double)
(framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j)));
@ -1064,26 +1096,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public Privacy getPrivacy() {
final boolean isUnlisted = playerResponse
.getObject("microformat")
.getObject("playerMicroformatRenderer")
.getBoolean("isUnlisted");
final boolean isUnlisted = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer").getBoolean("isUnlisted");
return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
}
@Nonnull
@Override
public String getCategory() {
return playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer")
.getString("category", EMPTY_STRING);
return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer")
.getString("category");
}
@Nonnull
@Override
public String getLicence() throws ParsingException {
final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows")
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getArray("rows")
.getObject(0).getObject("metadataRowRenderer");
final JsonArray contents = metadataRowRenderer.getArray("contents");
@ -1099,7 +1129,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords"));
return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails")
.getArray("keywords"));
}
@Nonnull
@ -1118,12 +1149,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) {
final String panelIdentifier = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
final String panelIdentifier = panels.getObject(i)
.getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier");
if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters")
|| panelIdentifier.equals("engagement-panel-macro-markers")) {
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
segmentsArray = panels.getObject(i)
.getObject("engagementPanelSectionListRenderer").getObject("content")
.getObject("macroMarkersListRenderer").getArray("contents");
break;
}
}
@ -1131,9 +1164,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (segmentsArray != null) {
final long duration = getLength();
for (final Object object : segmentsArray) {
final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer");
final JsonObject segmentJson = ((JsonObject) object)
.getObject("macroMarkersListItemRenderer");
final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint")
final int startTimeSeconds = segmentJson.getObject("onTap")
.getObject("watchEndpoint")
.getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) {
@ -1151,10 +1186,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails");
final JsonArray previewsArray = segmentJson.getObject("thumbnail")
.getArray("thumbnails");
if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the last position
final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url");
// Assume that the thumbnail with the highest resolution is at the
// last position
final String url = previewsArray
.getObject(previewsArray.size() - 1).getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url));
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -42,6 +42,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -57,18 +58,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
// @formatter:off
final byte[] body = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("browseId", "FEtrending")
.end().done().getBytes(UTF_8);
.done())
.getBytes(UTF_8);
// @formatter:on
initialData = getJsonPostResponse("browse", body, getExtractorLocalization());