From b74a39c1760721ecd68269cdcb0e1b2661f4dcb3 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sun, 1 Aug 2021 13:16:46 +0200 Subject: [PATCH] Reformat some code and don't use the clickTrackingParams in continuations of YouTube Music search results The clickTrackingParams of YouTube Music search results are not needed to get continuations. This commit removes their use, which may improve privacy. --- .../youtube/YoutubeThrottlingDecrypter.java | 2 + .../YoutubeMusicSearchExtractor.java | 172 +++++++++++------- .../newpipe/extractor/utils/StringUtils.java | 1 - 3 files changed, 111 insertions(+), 64 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index fb56e2c7b..2f241786e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -79,11 +79,13 @@ public class YoutubeThrottlingDecrypter { } } + @Nonnull private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) { final String functionBase = functionName + "=function"; return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";"; } + @Nonnull private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException { Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n", Pattern.DOTALL); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7aab9040c..f7d4b9fc3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -33,16 +34,18 @@ import static org.schabi.newpipe.extractor.utils.Utils.*; public class YoutubeMusicSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + public YoutubeMusicSearchExtractor(final StreamingService service, + final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, - ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + + youtubeMusicKeys[0]; final String params; @@ -68,7 +71,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } // @formatter:off - byte[] json = JsonWriter.string() + final byte[] json = JsonWriter.string() .object() .object("context") .object("client") @@ -104,11 +107,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, + json)); try { initialData = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } } @@ -122,20 +126,26 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer"); + final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, + "contents.tabbedSearchResultsRenderer.tabs").getObject(0), + "tabRenderer.content.sectionListRenderer.contents") + .getObject(0) + .getObject("itemSectionRenderer"); if (itemSectionRenderer.isEmpty()) { return ""; } final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") .getObject(0).getObject("didYouMeanRenderer"); - final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents") + .getObject(0) .getObject("showingResultsForRenderer"); if (!didYouMeanRenderer.isEmpty()) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } else if (!showingResultsForRenderer.isEmpty()) { - return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + return JsonUtils.getString(showingResultsForRenderer, + "correctedQueryEndpoint.searchEndpoint.query"); } else { return ""; } @@ -143,16 +153,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public boolean isCorrectedSearch() throws ParsingException { - final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer"); + final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, + "contents.tabbedSearchResultsRenderer.tabs").getObject(0), + "tabRenderer.content.sectionListRenderer.contents") + .getObject(0) + .getObject("itemSectionRenderer"); if (itemSectionRenderer.isEmpty()) { return false; } JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0); - final boolean corrected = firstContent - .has("didYouMeanRenderer") || firstContent.has("showingResultsForRenderer"); - return corrected; + return firstContent.has("didYouMeanRenderer") + || firstContent.has("showingResultsForRenderer"); } @Nonnull @@ -163,16 +176,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents"); + final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData, + "contents.tabbedSearchResultsRenderer.tabs").getObject(0), + "tabRenderer.content.sectionListRenderer.contents"); Page nextPage = null; - for (Object content : contents) { + for (final Object content : contents) { if (((JsonObject) content).has("musicShelfRenderer")) { - final JsonObject musicShelfRenderer = ((JsonObject) content).getObject("musicShelfRenderer"); + final JsonObject musicShelfRenderer = ((JsonObject) content) + .getObject("musicShelfRenderer"); collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents")); @@ -229,16 +245,18 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), + headers, json)); final JsonObject ajaxJson; try { ajaxJson = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } - final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents") + .getObject("musicShelfContinuation"); collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); final JsonArray continuations = musicShelfContinuation.getArray("continuations"); @@ -246,31 +264,32 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } - private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { + private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, + @Nonnull final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object item : videos) { + for (final Object item : videos) { final JsonObject info = ((JsonObject) item) .getObject("musicResponsiveListItemRenderer", null); if (info != null) { - final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING); + final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", + EMPTY_STRING); if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { - continue; // no info about video URL available + continue; // No info about video URL available } - final JsonObject flexColumnRenderer = info - .getArray("flexColumns") + final JsonObject flexColumnRenderer = info.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer"); - final JsonArray descriptionElements = flexColumnRenderer - .getObject("text") + final JsonArray descriptionElements = flexColumnRenderer.getObject("text") .getArray("runs"); final String searchType = getLinkHandler().getContentFilters().get(0); if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { @Override public String getUrl() throws ParsingException { - final String id = info.getObject("playlistItemData").getString("videoId"); + final String id = info.getObject("playlistItemData") + .getString("videoId"); if (!isNullOrEmpty(id)) { return "https://music.youtube.com/watch?v=" + id; } @@ -279,8 +298,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String name = getTextFromObject(info.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); if (!isNullOrEmpty(name)) { return name; } @@ -310,24 +331,34 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) { - JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); - for (Object item : items) { - final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) { - return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + JsonArray items = info.getObject("menu").getObject("menuRenderer") + .getArray("items"); + for (final Object item : items) { + final JsonObject menuNavigationItemRenderer = + ((JsonObject) item).getObject( + "menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon") + .getString("iconType", EMPTY_STRING) + .equals("ARTIST")) { + return getUrlFromNavigationEndpoint( + menuNavigationItemRenderer + .getObject("navigationEndpoint")); } } return null; } else { - final JsonObject navigationEndpointHolder = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + final JsonObject navigationEndpointHolder = info + .getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text").getArray("runs").getObject(0); if (!navigationEndpointHolder.has("navigationEndpoint")) return null; - final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint")); + final String url = getUrlFromNavigationEndpoint( + navigationEndpointHolder.getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { return url; @@ -369,13 +400,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + final JsonArray thumbnails = info.getObject("thumbnail") + .getObject("musicThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + final String url = thumbnails.getObject(thumbnails.size() - 1) + .getString("url"); return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get thumbnail url", e); } } @@ -385,21 +418,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + final JsonArray thumbnails = info.getObject("thumbnail") + .getObject("musicThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + final String url = thumbnails.getObject(thumbnails.size() - 1) + .getString("url"); return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get thumbnail url", e); } } @Override public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String name = getTextFromObject(info.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); if (!isNullOrEmpty(name)) { return name; } @@ -408,7 +445,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + final String url = getUrlFromNavigationEndpoint(info + .getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { return url; } @@ -417,8 +455,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public long getSubscriberCount() throws ParsingException { - final String subscriberCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String subscriberCount = getTextFromObject(info + .getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); if (!isNullOrEmpty(subscriberCount)) { try { return Utils.mixedNumberWordToLong(subscriberCount); @@ -445,21 +485,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + final JsonArray thumbnails = info.getObject("thumbnail") + .getObject("musicThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + final String url = thumbnails.getObject(thumbnails.size() - 1) + .getString("url"); return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get thumbnail url", e); } } @Override public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + final String name = getTextFromObject(info.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); if (!isNullOrEmpty(name)) { return name; } @@ -512,7 +556,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { if (searchType.equals(MUSIC_ALBUMS)) { return ITEM_COUNT_UNKNOWN; } - final String count = descriptionElements.getObject(2).getString("text"); + final String count = descriptionElements.getObject(2) + .getString("text"); if (!isNullOrEmpty(count)) { if (count.contains("100+")) { return ITEM_COUNT_MORE_THAN_100; @@ -528,18 +573,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } } - private Page getNextPageFrom(final JsonArray continuations) throws ParsingException, - IOException, ReCaptchaException { + @Nullable + private Page getNextPageFrom(final JsonArray continuations) + throws IOException, ParsingException, ReCaptchaException { if (isNullOrEmpty(continuations)) { return null; } - final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final JsonObject nextContinuationData = continuations.getObject(0) + .getObject("nextContinuationData"); final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation - + "&continuation=" + continuation + "&itct=" + clickTrackingParams + "&alt=json" - + "&key=" + YoutubeParsingHelper.getYoutubeMusicKey()[0]); + + "&continuation=" + continuation + "&alt=json" + "&key=" + + YoutubeParsingHelper.getYoutubeMusicKey()[0]); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java index a0429156f..ea61bbfbc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.utils; - import javax.annotation.Nonnull; public class StringUtils {