diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index fba78cf34..46763f384 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -63,7 +63,7 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - private static final String HARDCODED_CLIENT_VERSION = "2.20210408.08.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20210413.07.00"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static String clientVersion; private static String key; @@ -729,7 +729,7 @@ public class YoutubeParsingHelper { return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); } - public static JsonBuilder prepareJsonBuilder() + public static JsonBuilder prepareJsonBuilder(final String contentCountry) throws IOException, ExtractionException { // @formatter:off return JsonObject.builder() @@ -737,6 +737,8 @@ public class YoutubeParsingHelper { .object("client") .value("clientName", "1") .value("clientVersion", getClientVersion()) + .value("hl", "en-GB") + .value("gl", contentCountry) .end() .end(); // @formatter:on diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index a802e531d..5f1a07f20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -73,20 +73,23 @@ public class YoutubeChannelExtractor extends ChannelExtractor { */ private String redirectedChannelId; - public YoutubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { + public YoutubeChannelExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { final String channel_path = super.getId(); final String[] channelInfo = channel_path.split("/"); + final String contentCountry = getExtractorContentCountry().getCountryCode(); String id = ""; // If the url is an URL which is not a /channel URL, we need to use the // navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we // couldn't get information about the channel associated with this URL, if there is one. if (!channelInfo[0].equals("channel")) { - final byte[] body = JsonWriter.string(prepareJsonBuilder() + final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry) .value("url", "https://www.youtube.com/" + channel_path) .done()) .getBytes(UTF_8); @@ -131,7 +134,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { int level = 0; while (level < 3) { - final byte[] body = JsonWriter.string(prepareJsonBuilder() + final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry) .value("browseId", id) .value("params", "EgZ2aWRlb3M%3D") // equals to videos .done()) @@ -336,7 +339,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -361,7 +365,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException { + private Page getNextPageFrom(final JsonObject continuations) throws IOException, + ExtractionException { if (isNullOrEmpty(continuations)) { return null; } @@ -370,7 +375,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String continuation = continuationEndpoint.getObject("continuationCommand") .getString("token"); - final byte[] body = JsonWriter.string(prepareJsonBuilder() + final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry() + .getCountryCode()) .value("continuation", continuation) .done()) .getBytes(UTF_8); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 9dfe92c3a..0d53be6a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -40,7 +40,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - private JsonArray initialAjaxJson; private JsonObject initialData; private JsonObject playlistInfo; @@ -49,8 +48,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final byte[] body = JsonWriter.string(prepareJsonBuilder() + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { + final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry() + .getCountryCode()) .value("browseId", "VL" + getId()) .value("params", "wgYCCAA%3D") // show unavailable videos .done()) @@ -63,15 +64,18 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } private JsonObject getUploaderInfo() throws ParsingException { - final JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); + final JsonArray items = initialData.getObject("sidebar") + .getObject("playlistSidebarRenderer").getArray("items"); - JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + JsonObject videoOwner = items.getObject(1) + .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); if (videoOwner.has("videoOwnerRenderer")) { return videoOwner.getObject("videoOwnerRenderer"); } // we might want to create a loop here instead of using duplicated code - videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + videoOwner = items.getObject(items.size()) + .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); if (videoOwner.has("videoOwnerRenderer")) { return videoOwner.getObject("videoOwnerRenderer"); } @@ -80,9 +84,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject getPlaylistInfo() throws ParsingException { try { - return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items") - .getObject(0).getObject("playlistSidebarPrimaryInfoRenderer"); - } catch (Exception e) { + return initialData.getObject("sidebar").getObject("playlistSidebarRenderer") + .getArray("items").getObject(0) + .getObject("playlistSidebarPrimaryInfoRenderer"); + } catch (final Exception e) { throw new ParsingException("Could not get PlaylistInfo", e); } } @@ -122,7 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public String getUploaderUrl() throws ParsingException { try { return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint")); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } } @@ -131,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public String getUploaderName() throws ParsingException { try { return getTextFromObject(getUploaderInfo().getObject("title")); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } } @@ -142,7 +147,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } } @@ -157,7 +162,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get video count from playlist", e); } } @@ -186,18 +191,21 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Page nextPage = null; - final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents"); + final JsonArray contents = initialData.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0) + .getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents"); if (contents.getObject(0).has("playlistSegmentRenderer")) { for (final Object segment : contents) { if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) { collectTrailerFrom(collector, ((JsonObject) segment)); - } else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) { - collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer") - .getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents")); + } else if (((JsonObject) segment).getObject("playlistSegmentRenderer") + .has("videoList")) { + collectStreamsFrom(collector, ((JsonObject) segment) + .getObject("playlistSegmentRenderer").getObject("videoList") + .getObject("playlistVideoListRenderer").getArray("contents")); } } @@ -214,7 +222,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -235,7 +244,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException { + private Page getNextPageFrom(final JsonArray contents) throws IOException, + ExtractionException { if (isNullOrEmpty(contents)) { return null; } @@ -248,7 +258,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .getObject("continuationCommand") .getString("token"); - final byte[] body = JsonWriter.string(prepareJsonBuilder() + final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry() + .getCountryCode()) .value("continuation", continuation) .done()) .getBytes(UTF_8); @@ -260,12 +271,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } - private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { + private void collectStreamsFrom(final StreamInfoItemsCollector collector, + final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (final Object video : videos) { if (((JsonObject) video).has("playlistVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video) + .getObject("playlistVideoRenderer"), timeAgoParser) { @Override public long getViewCount() { return -1; @@ -294,11 +307,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() { - final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse") + return ""; + /*final JsonArray thumbnails = initialAjaxJson.getObject(1) + .getObject("playerResponse") .getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - return fixThumbnailUrl(url); + return fixThumbnailUrl(url);*/ } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index d917b0914..d96a5f3c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -50,13 +50,16 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + public YoutubeSearchExtractor(final StreamingService service, + final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { final String query = super.getSearchString(); + final String contentCountry = getExtractorContentCountry().getCountryCode(); // Get the search parameter of the request final List contentFilters = super.getLinkHandler().getContentFilters(); @@ -70,13 +73,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { final byte[] body; if (!isNullOrEmpty(params)) { - body = JsonWriter.string(prepareJsonBuilder() + body = JsonWriter.string(prepareJsonBuilder(contentCountry) .value("query", query) .value("params", params) .done()) .getBytes(UTF_8); } else { - body = JsonWriter.string(prepareJsonBuilder() + body = JsonWriter.string(prepareJsonBuilder(contentCountry) .value("query", query) .done()) .getBytes(UTF_8); @@ -100,11 +103,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("itemSectionRenderer"); final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("didYouMeanRenderer"); - final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents") + .getObject(0) .getObject("showingResultsForRenderer"); if (!didYouMeanRenderer.isEmpty()) { - return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + return JsonUtils.getString(didYouMeanRenderer, + "correctedQueryEndpoint.searchEndpoint.query"); } else if (showingResultsForRenderer != null) { return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } else { @@ -126,7 +131,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { public List getMetaInfo() throws ParsingException { return YoutubeParsingHelper.getMetaInfo( initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents")); + .getObject("primaryContents").getObject("sectionListRenderer") + .getArray("contents")); } @Nonnull @@ -134,20 +140,23 @@ public class YoutubeSearchExtractor extends SearchExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + final JsonArray sections = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents"); Page nextPage = null; for (final Object section : sections) { if (((JsonObject) section).has("itemSectionRenderer")) { - final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer"); + final JsonObject itemSectionRenderer = ((JsonObject) section) + .getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations")); } else if (((JsonObject) section).has("continuationItemRenderer")) { - nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer")); + nextPage = getNewNextPageFrom(((JsonObject) section) + .getObject("continuationItemRenderer")); } } @@ -155,7 +164,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -174,29 +184,15 @@ public class YoutubeSearchExtractor extends SearchExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } else { // @formatter:off - final byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .value("clientName", "1") - .value("clientVersion", getClientVersion()) - .value("utcOffsetMinutes", 0) - .end() - .object("request").end() - .object("user").end() - .end() + final byte[] json = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry() + .getCountryCode()) .value("continuation", page.getId()) - .end().done().getBytes(UTF_8); + .done()) + .getBytes(UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("Origin", Collections.singletonList("https://www.youtube.com")); - headers.put("Referer", Collections.singletonList(this.getUrl())); - headers.put("Content-Type", Collections.singletonList("application/json")); - - final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post( + page.getUrl(), new HashMap<>(), json)); final JsonObject ajaxJson; try { @@ -206,16 +202,21 @@ public class YoutubeSearchExtractor extends SearchExtractor { } final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") - .getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems"); + .getObject(0).getObject("appendContinuationItemsAction") + .getArray("continuationItems"); - final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents"); + final JsonArray contents = continuationItems.getObject(0) + .getObject("itemSectionRenderer").getArray("contents"); collectStreamsFrom(collector, contents); - return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer"))); + return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1) + .getObject("continuationItemRenderer"))); } } - private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray contents) throws NothingFoundException, ParsingException { + private void collectStreamsFrom(final InfoItemsSearchCollector collector, + final JsonArray contents) throws NothingFoundException, + ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object content : contents) { @@ -224,11 +225,14 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new NothingFoundException(getTextFromObject( item.getObject("backgroundPromoRenderer").getObject("bodyText"))); } else if (item.has("videoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(item.getObject("videoRenderer"), timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor(item + .getObject("videoRenderer"), timeAgoParser)); } else if (item.has("channelRenderer")) { - collector.commit(new YoutubeChannelInfoItemExtractor(item.getObject("channelRenderer"))); + collector.commit(new YoutubeChannelInfoItemExtractor(item + .getObject("channelRenderer"))); } else if (item.has("playlistRenderer")) { - collector.commit(new YoutubePlaylistInfoItemExtractor(item.getObject("playlistRenderer"))); + collector.commit(new YoutubePlaylistInfoItemExtractor(item + .getObject("playlistRenderer"))); } } } @@ -238,15 +242,18 @@ public class YoutubeSearchExtractor extends SearchExtractor { return null; } - final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final JsonObject nextContinuationData = continuations.getObject(0) + .getObject("nextContinuationData"); final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + final String clickTrackingParams = nextContinuationData + .getString("clickTrackingParams"); - return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams); + return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + + continuation + "&itct=" + clickTrackingParams); } - private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { + private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, + ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 53d9449e5..4acc1fb74 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,6 +4,11 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -12,8 +17,15 @@ import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.*; +import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; +import org.schabi.newpipe.extractor.exceptions.PaidContentException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.PrivateContentException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; @@ -120,8 +132,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - final JsonObject micro = - playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + final JsonObject micro = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer"); if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) { return micro.getString("uploadDate"); } else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) { @@ -140,11 +152,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { - String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) + .startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) + .substring(10); try { // Premiered 20 hours ago - TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor( + Localization.fromLocalizationCode("en")); OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime(); return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime); } catch (final Exception ignored) { @@ -159,8 +174,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - // TODO: this parses English formatted dates only, we need a better approach to parse the textual date - LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")), + // TODO: this parses English formatted dates only, we need a better approach to parse + // the textual date + LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer() + .getObject("dateText")), DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (final Exception ignored) { @@ -185,7 +202,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getThumbnailUrl() throws ParsingException { assertPageFetched(); try { - JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); + JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail") + .getArray("thumbnails"); // the last thumbnail is the one with the highest resolution String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); @@ -202,13 +220,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // description with more info on links try { - String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + String description = getTextFromObject(getVideoSecondaryInfoRenderer() + .getObject("description"), true); if (!isNullOrEmpty(description)) return new Description(description, Description.HTML); } catch (final ParsingException ignored) { // age-restricted videos cause a ParsingException here } - String description = playerResponse.getObject("videoDetails").getString("shortDescription"); + String description = playerResponse.getObject("videoDetails") + .getString("shortDescription"); if (description == null) { final JsonObject descriptionObject = playerResponse.getObject("microformat") .getObject("playerMicroformatRenderer").getObject("description"); @@ -289,22 +309,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { assertPageFetched(); - String views = null; - - try { - views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount")); - } catch (final ParsingException ignored) { - // age-restricted videos cause a ParsingException here - } - - if (isNullOrEmpty(views)) { - views = playerResponse.getObject("videoDetails").getString("viewCount"); - - if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count"); - } - - if (views.toLowerCase().contains("no views")) return 0; + final String views = playerResponse.getObject("videoDetails").getString("viewCount"); + if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count"); return Long.parseLong(Utils.removeNonDigitCharacters(views)); } @@ -320,13 +326,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (final NullPointerException e) { // if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { - throw new ParsingException("Ratings are enabled even though the like button is missing", e); + throw new ParsingException( + "Ratings are enabled even though the like button is missing", e); } return -1; } return Integer.parseInt(Utils.removeNonDigitCharacters(likesString)); } catch (final NumberFormatException nfe) { - throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe); + throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", + nfe); } catch (final Exception e) { if (getAgeLimit() == NO_AGE_LIMIT) { throw new ParsingException("Could not get like count", e); @@ -338,6 +346,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getDislikeCount() throws ParsingException { assertPageFetched(); + String dislikesString = ""; try { try { @@ -346,13 +355,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (final NullPointerException e) { // if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { - throw new ParsingException("Ratings are enabled even though the dislike button is missing", e); + throw new ParsingException( + "Ratings are enabled even though the dislike button is missing", e); } return -1; } return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString)); } catch (final NumberFormatException nfe) { - throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe); + throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", + nfe); } catch (final Exception e) { if (getAgeLimit() == NO_AGE_LIMIT) { throw new ParsingException("Could not get dislike count", e); @@ -366,16 +377,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { assertPageFetched(); - try { - final String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() - .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); - if (!isNullOrEmpty(uploaderUrl)) { - return uploaderUrl; - } - } catch (final ParsingException ignored) { - // age-restricted videos cause a ParsingException here - } - final String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); if (!isNullOrEmpty(uploaderId)) { return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); @@ -389,19 +390,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderName() throws ParsingException { assertPageFetched(); - String uploaderName = null; - - try { - uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") - .getObject("videoOwnerRenderer").getObject("title")); - } catch (final ParsingException ignored) { - } - - if (isNullOrEmpty(uploaderName)) { - uploaderName = playerResponse.getObject("videoDetails").getString("author"); - - if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name"); - } + // Don't use the name in the videoSecondaryRenderer object to get real name of the uploader + // The difference between the real name of the channel and the displayed name is especially + // visible for music channels and autogenerated channels. + final String uploaderName = playerResponse.getObject("videoDetails").getString("author"); + if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name"); return uploaderName; } @@ -422,8 +415,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { String url = null; try { - url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + url = getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); } catch (final ParsingException ignored) { // age-restricted videos cause a ParsingException here } @@ -471,11 +465,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } if (!dashManifestUrl.contains("/signature/")) { - String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); + String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", + dashManifestUrl); final String deobfuscatedSig; deobfuscatedSig = deobfuscateSignature(obfuscatedSig); - dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig); + dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, + "/signature/" + deobfuscatedSig); } return dashManifestUrl; @@ -503,7 +499,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { - for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { + for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, + ItagItem.ItagType.AUDIO).entrySet()) { final ItagItem itag = entry.getValue(); String url = entry.getKey(); url = throttlingDecrypter.apply(url); @@ -527,7 +524,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { - for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { + for (final Map.Entry entry : getItags(FORMATS, + ItagItem.ItagType.VIDEO).entrySet()) { final ItagItem itag = entry.getValue(); String url = entry.getKey(); url = throttlingDecrypter.apply(url); @@ -551,7 +549,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { - for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, + ItagItem.ItagType.VIDEO_ONLY).entrySet()) { final ItagItem itag = entry.getValue(); String url = entry.getKey(); url = throttlingDecrypter.apply(url); @@ -620,27 +619,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM; } - @Nullable - private StreamInfoItemExtractor getNextStream() throws ExtractionException { - try { - final JsonObject firstWatchNextItem = initialData.getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("secondaryResults") - .getObject("secondaryResults").getArray("results").getObject(0); - - if (!firstWatchNextItem.has("compactAutoplayRenderer")) { - // there is no "next" stream - return null; - } - - final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer") - .getArray("contents").getObject(0).getObject("compactVideoRenderer"); - - return new YoutubeStreamInfoItemExtractor(videoInfo, getTimeAgoParser()); - } catch (final Exception e) { - throw new ParsingException("Could not get next video", e); - } - } - @Nullable @Override public StreamInfoItemsCollector getRelatedItems() throws ExtractionException { @@ -651,21 +629,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector( + getServiceId()); - final StreamInfoItemExtractor nextStream = getNextStream(); - if (nextStream != null) { - collector.commit(nextStream); - } - - final JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("secondaryResults").getObject("secondaryResults").getArray("results"); + final JsonArray results = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("secondaryResults") + .getObject("secondaryResults").getArray("results"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (final Object ul : results) { if (((JsonObject) ul).has("compactVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul) + .getObject("compactVideoRenderer"), timeAgoParser)); } } return collector; @@ -680,9 +656,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getErrorMessage() { try { - return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse") - .getObject("playabilityStatus").getObject("errorScreen") - .getObject("playerErrorMessageRenderer").getObject("reason")); + return getTextFromObject(playerResponse.getObject("playabilityStatus") + .getObject("errorScreen").getObject("playerErrorMessageRenderer") + .getObject("reason")); } catch (final ParsingException | NullPointerException e) { return null; // no error message } @@ -737,7 +713,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { youtubePlayerResponse = playerResponse; } - JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse : playerResponse) + JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse + : playerResponse) .getObject("playabilityStatus"); String status = playabilityStatus.getString("status"); // If status exist, and is not "OK", throw the specific exception based on error message @@ -755,8 +732,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new PrivateContentException("This video is private."); } } else if (reason.equals("Sign in to confirm your age")) { - // No streams can be fetched, therefore thrown an AgeRestrictedContentException explicitly. - throw new AgeRestrictedContentException("This age-restricted video cannot be watched."); + // No streams can be fetched, therefore thrown an AgeRestrictedContentException + // explicitly. + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched."); } } if (status.equalsIgnoreCase("unplayable")) { @@ -772,7 +751,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new PaidContentException("This video is only available for members of the channel of this video"); } if (reason.equals("Video unavailable")) { - final String detailedErrorMessage = playabilityStatus.getObject("errorScreen") + final String detailedErrorMessage = playabilityStatus + .getObject("errorScreen") .getObject("playerErrorMessageRenderer") .getObject("subreason") .getArray("runs") @@ -780,7 +760,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("text"); if (detailedErrorMessage != null) { if (detailedErrorMessage.equals("The uploader has not made this video available in your country.")) { - throw new GeographicRestrictionException("This video is not available in user's country."); + throw new GeographicRestrictionException( + "This video is not available in user's country."); } } } @@ -792,9 +773,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException { - final String videoInfoUrl = getVideoInfoUrl(getId()); - final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization()); - videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody())); + final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl(); + final String videoInfoUrl = getVideoInfoUrl(getId(), sts); + final String infoPageResponse = NewPipe.getDownloader() + .get(videoInfoUrl, getExtractorLocalization()).responseBody(); + videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); try { playerResponse = JsonParser.object().from(videoInfoPage.get("player_response")); @@ -804,6 +787,39 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull + private String getEmbeddedInfoStsAndStorePlayerJsUrl() { + try { + // Don't provide a video id to get a smaller response (around 9kb instead of 21 kb) + final String embedUrl = "https://www.youtube.com/embed/"; + final String embedPageContent = NewPipe.getDownloader() + .get(embedUrl, getExtractorLocalization()).responseBody(); + + try { + final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; + playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent) + .replace("\\", "").replace("\"", ""); + } catch (final Parser.RegexException ex) { + // playerJsUrl is still available in the file, just somewhere else TODO + // it is ok not to find it, see how that's handled in getDeobfuscationCode() + final Document doc = Jsoup.parse(embedPageContent); + final Elements elems = doc.select("script").attr("name", "player_ias/base"); + for (final Element elem : elems) { + if (elem.attr("src").contains("base.js")) { + playerJsUrl = elem.attr("src"); + break; + } + } + } + + // Get embed sts + return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent); + } catch (final Exception i) { + // if it fails we simply reply with no sts as then it does not seem to be necessary + return ""; + } + } + private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { Parser.RegexException exception = null; for (final String regex : REGEXES) { @@ -815,7 +831,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } } - throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); + throw new DeobfuscateException( + "Could not find deobfuscate function with any of the given patterns.", exception); } private String loadDeobfuscationCode() @@ -827,17 +844,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String functionPattern = "(" + deobfuscationFunctionName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; - final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; + final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, + playerCode) + ";"; final String helperObjectName = - Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction); + Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", + deobfuscateFunction); final String helperPattern = - "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; + "(var " + helperObjectName.replace("$", "\\$") + + "=\\{.+?\\}\\};)"; final String helperObject = Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); final String callerFunction = - "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}"; + "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + + deobfuscationFunctionName + "(a);}"; return helperObject + deobfuscateFunction + callerFunction; } catch (final Exception e) { @@ -879,13 +900,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; - JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("results").getObject("results").getArray("contents"); + final JsonArray contents = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("results").getObject("results") + .getArray("contents"); JsonObject videoPrimaryInfoRenderer = null; for (final Object content : contents) { if (((JsonObject) content).has("videoPrimaryInfoRenderer")) { - videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); + videoPrimaryInfoRenderer = ((JsonObject) content) + .getObject("videoPrimaryInfoRenderer"); break; } } @@ -901,13 +924,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; - JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("results").getObject("results").getArray("contents"); + final JsonArray contents = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("results").getObject("results") + .getArray("contents"); JsonObject videoSecondaryInfoRenderer = null; for (final Object content : contents) { if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { - videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); + videoSecondaryInfoRenderer = ((JsonObject) content) + .getObject("videoSecondaryInfoRenderer"); break; } } @@ -921,11 +946,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Nonnull - private static String getVideoInfoUrl(final String id) { + private static String getVideoInfoUrl(final String id, final String sts) { // TODO: Try parsing embedded_player_response first - return "https://www.youtube.com/get_video_info?video_id=" + id + - "&eurl=https://youtube.googleapis.com/v/" + id + - "&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en"; + return "https://www.youtube.com/get_video_info?" + "video_id=" + id + + "&html5=1&eurl=https://youtube.googleapis.com/v/" + id + + "&sts=" + sts + "&ps=default&gl=US&hl=en"; } private Map getItags(final String streamingDataKey, @@ -969,15 +994,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject initRange = formatData.getObject("initRange"); JsonObject indexRange = formatData.getObject("indexRange"); String mimeType = formatData.getString("mimeType", EMPTY_STRING); - String codec = mimeType.contains("codecs") ? mimeType.split("\"")[1] : EMPTY_STRING; + String codec = mimeType.contains("codecs") + ? mimeType.split("\"")[1] : EMPTY_STRING; itagItem.setBitrate(formatData.getInt("bitrate")); itagItem.setWidth(formatData.getInt("width")); itagItem.setHeight(formatData.getInt("height")); - itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1"))); - itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1"))); - itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1"))); - itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1"))); + itagItem.setInitStart(Integer.parseInt(initRange.getString("start", + "-1"))); + itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", + "-1"))); + itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", + "-1"))); + itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", + "-1"))); itagItem.fps = formatData.getInt("fps"); itagItem.setQuality(formatData.getString("quality")); itagItem.setCodec(codec); @@ -1027,10 +1057,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { final int totalCount = Integer.parseInt(parts[2]); final int framesPerPageX = Integer.parseInt(parts[3]); final int framesPerPageY = Integer.parseInt(parts[4]); - final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; + final String baseUrl = url.replace("$L", String.valueOf(i - 1)) + .replace("$N", parts[6]) + "&sigh=" + parts[7]; final List urls; if (baseUrl.contains("$M")) { - final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); + final int totalPages = (int) Math.ceil(totalCount / (double) + (framesPerPageX * framesPerPageY)); urls = new ArrayList<>(totalPages); for (int j = 0; j < totalPages; j++) { urls.add(baseUrl.replace("$M", String.valueOf(j))); @@ -1064,26 +1096,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public Privacy getPrivacy() { - final boolean isUnlisted = playerResponse - .getObject("microformat") - .getObject("playerMicroformatRenderer") - .getBoolean("isUnlisted"); + final boolean isUnlisted = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer").getBoolean("isUnlisted"); return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC; } @Nonnull @Override public String getCategory() { - return playerResponse.getObject("microformat") - .getObject("playerMicroformatRenderer") - .getString("category", EMPTY_STRING); + return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer") + .getString("category"); } @Nonnull @Override public String getLicence() throws ParsingException { final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() - .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows") + .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer") + .getArray("rows") .getObject(0).getObject("metadataRowRenderer"); final JsonArray contents = metadataRowRenderer.getArray("contents"); @@ -1099,7 +1129,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords")); + return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails") + .getArray("keywords")); } @Nonnull @@ -1118,12 +1149,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Search for correct panel containing the data for (int i = 0; i < panels.size(); i++) { - final String panelIdentifier = panels.getObject(i).getObject("engagementPanelSectionListRenderer") + final String panelIdentifier = panels.getObject(i) + .getObject("engagementPanelSectionListRenderer") .getString("panelIdentifier"); if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters") || panelIdentifier.equals("engagement-panel-macro-markers")) { - segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer") - .getObject("content").getObject("macroMarkersListRenderer").getArray("contents"); + segmentsArray = panels.getObject(i) + .getObject("engagementPanelSectionListRenderer").getObject("content") + .getObject("macroMarkersListRenderer").getArray("contents"); break; } } @@ -1131,9 +1164,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (segmentsArray != null) { final long duration = getLength(); for (final Object object : segmentsArray) { - final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer"); + final JsonObject segmentJson = ((JsonObject) object) + .getObject("macroMarkersListItemRenderer"); - final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint") + final int startTimeSeconds = segmentJson.getObject("onTap") + .getObject("watchEndpoint") .getInt("startTimeSeconds", -1); if (startTimeSeconds == -1) { @@ -1151,10 +1186,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { final StreamSegment segment = new StreamSegment(title, startTimeSeconds); segment.setUrl(getUrl() + "?t=" + startTimeSeconds); if (segmentJson.has("thumbnail")) { - final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails"); + final JsonArray previewsArray = segmentJson.getObject("thumbnail") + .getArray("thumbnails"); if (!previewsArray.isEmpty()) { - // Assume that the thumbnail with the highest resolution is at the last position - final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url"); + // Assume that the thumbnail with the highest resolution is at the + // last position + final String url = previewsArray + .getObject(previewsArray.size() - 1).getString("url"); segment.setPreviewUrl(fixThumbnailUrl(url)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index b02f60757..d63024c2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -42,6 +42,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -57,18 +58,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { // @formatter:off - final byte[] body = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .value("clientName", "1") - .value("clientVersion", getClientVersion()) - .end() - .end() + final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry() + .getCountryCode()) .value("browseId", "FEtrending") - .end().done().getBytes(UTF_8); + .done()) + .getBytes(UTF_8); // @formatter:on initialData = getJsonPostResponse("browse", body, getExtractorLocalization());