From 27dc1b1f5063c721bce14e8f7db152cd24b577a2 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 4 Apr 2024 23:33:41 +0200 Subject: [PATCH] [YouTube] Remove usage of API keys for InnerTube requests, bump versions The API keys are not used anymore by official clients in almost all cases (still used by the Android app until it gets a configuration) for all requests we made. Clients and device OS versions have been bumped to their latest stable version known. Methods and fields related to API keys have been renamed or deleted if they're no longer relevant. --- .../youtube/YoutubeParsingHelper.java | 200 ++++++------------ .../YoutubeChannelTabExtractor.java | 5 +- .../YoutubeMixPlaylistExtractor.java | 9 +- .../YoutubeMusicSearchExtractor.java | 21 +- .../extractors/YoutubePlaylistExtractor.java | 4 +- .../extractors/YoutubeSearchExtractor.java | 7 +- .../YoutubeDashManifestCreatorsTest.java | 3 +- .../YoutubeMixPlaylistExtractorTest.java | 13 +- .../youtube/YoutubeParsingHelperTest.java | 12 +- .../services/youtube/YoutubeTestsUtils.java | 2 +- 10 files changed, 91 insertions(+), 185 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b6ea901d3..7acfeb46e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -100,17 +100,17 @@ public final class YoutubeParsingHelper { * sizes. * *

- * Sent in query parameters of the requests, after the API key. + * Sent in query parameters of the requests. *

**/ - public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false"; + public static final String DISABLE_PRETTY_PRINT_PARAMETER = "prettyPrint=false"; /** * A parameter sent by official clients named {@code contentPlaybackNonce}. * *

- * It is sent by official clients on videoplayback requests, and by all clients (except the - * {@code WEB} one to the player requests. + * It is sent by official clients on videoplayback requests and InnerTube player requests in + * most cases. *

* *

@@ -148,13 +148,7 @@ public final class YoutubeParsingHelper { * The client version for InnerTube requests with the {@code WEB} client, used as the last * fallback if the extraction of the real one failed. */ - private static final String HARDCODED_CLIENT_VERSION = "2.20231208.01.00"; - - /** - * The InnerTube API key which should be used by YouTube's desktop website, used as a fallback - * if the extraction of the real one failed. - */ - private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + private static final String HARDCODED_CLIENT_VERSION = "2.20240410.01.00"; /** * The hardcoded client version of the Android app used for InnerTube requests with this @@ -165,17 +159,10 @@ public final class YoutubeParsingHelper { * such as APKMirror. *

*/ - private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "18.48.37"; + private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.13.36"; /** - * The InnerTube API key used by the {@code ANDROID} client. Found with the help of - * reverse-engineering app network requests. - */ - private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; - - /** - * The hardcoded client version of the iOS app used for InnerTube requests with this - * client. + * The hardcoded client version of the iOS app used for InnerTube requests with this client. * *

* It can be extracted by getting the latest release version of the app on @@ -183,42 +170,34 @@ public final class YoutubeParsingHelper { * Store page of the YouTube app, in the {@code What’s New} section. *

*/ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "18.48.3"; - - /** - * The InnerTube API key used by the {@code iOS} client. Found with the help of - * reverse-engineering app network requests. - */ - private static final String IOS_YOUTUBE_KEY = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc"; + private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.14.3"; /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. */ private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0"; + /** + * The hardcoded client version used for InnerTube requests with the YouTube Music desktop + * client. + */ + private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240403.01.00"; + private static String clientVersion; - private static String key; - private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY = - {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20231204.01.00"}; - private static String[] youtubeMusicKey; + private static String youtubeMusicClientVersion; - private static boolean keyAndVersionExtracted = false; + private static boolean clientVersionExtracted = false; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private static Optional hardcodedClientVersionAndKeyValid = Optional.empty(); + private static Optional hardcodedClientVersionValid = Optional.empty(); private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES = {"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", "innertube_context_client_version\":\"([0-9\\.]+?)\"", "client.version=([0-9\\.]+)"}; - private static final String[] INNERTUBE_API_KEY_REGEXES = - {"INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", - "innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\""}; private static final String[] INITIAL_DATA_REGEXES = {"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});"}; - private static final String INNERTUBE_CLIENT_NAME_REGEX = - "INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),"; private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -518,10 +497,10 @@ public final class YoutubeParsingHelper { } } - public static boolean areHardcodedClientVersionAndKeyValid() + public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException { - if (hardcodedClientVersionAndKeyValid.isPresent()) { - return hardcodedClientVersionAndKeyValid.get(); + if (hardcodedClientVersionValid.isPresent()) { + return hardcodedClientVersionValid.get(); } // @formatter:off final byte[] body = JsonWriter.string() @@ -555,20 +534,20 @@ public final class YoutubeParsingHelper { // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) final Response response = getDownloader().postWithContentTypeJson( - YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "guide?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body); final String responseBody = response.responseBody(); final int responseCode = response.responseCode(); - hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000 + hardcodedClientVersionValid = Optional.of(responseBody.length() > 5000 && responseCode == 200); // Ensure to have a valid response - return hardcodedClientVersionAndKeyValid.get(); + return hardcodedClientVersionValid.get(); } - private static void extractClientVersionAndKeyFromSwJs() + private static void extractClientVersionFromSwJs() throws IOException, ExtractionException { - if (keyAndVersionExtracted) { + if (clientVersionExtracted) { return; } final String url = "https://www.youtube.com/sw.js"; @@ -577,18 +556,17 @@ public final class YoutubeParsingHelper { try { clientVersion = getStringResultFromRegexArray(response, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); } catch (final Parser.RegexException e) { throw new ParsingException("Could not extract YouTube WEB InnerTube client version " - + "and API key from sw.js", e); + + "from sw.js", e); } - keyAndVersionExtracted = true; + clientVersionExtracted = true; } - private static void extractClientVersionAndKeyFromHtmlSearchResultsPage() + private static void extractClientVersionFromHtmlSearchResultsPage() throws IOException, ExtractionException { - // Don't extract the client version and the InnerTube key if it has been already extracted - if (keyAndVersionExtracted) { + // Don't extract the InnerTube client version if it has been already extracted + if (clientVersionExtracted) { return; } @@ -622,18 +600,6 @@ public final class YoutubeParsingHelper { serviceTrackingParamsStream, "ECATCHER", "client.version"); } - try { - key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); - } catch (final Parser.RegexException ignored) { - } - - if (isNullOrEmpty(key)) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube API key from HTML search results page"); - // CHECKSTYLE:ON - } - if (clientVersion == null) { throw new ParsingException( // CHECKSTYLE:OFF @@ -641,7 +607,7 @@ public final class YoutubeParsingHelper { // CHECKSTYLE:ON } - keyAndVersionExtracted = true; + clientVersionExtracted = true; } @Nullable @@ -676,17 +642,17 @@ public final class YoutubeParsingHelper { // JavaScript service worker, then from HTML search results page as a fallback, to prevent // fingerprinting based on the client version used try { - extractClientVersionAndKeyFromSwJs(); + extractClientVersionFromSwJs(); } catch (final Exception e) { - extractClientVersionAndKeyFromHtmlSearchResultsPage(); + extractClientVersionFromHtmlSearchResultsPage(); } - if (keyAndVersionExtracted) { + if (clientVersionExtracted) { return clientVersion; } // Fallback to the hardcoded one if it is valid - if (areHardcodedClientVersionAndKeyValid()) { + if (isHardcodedClientVersionValid()) { clientVersion = HARDCODED_CLIENT_VERSION; return clientVersion; } @@ -694,39 +660,6 @@ public final class YoutubeParsingHelper { throw new ExtractionException("Could not get YouTube WEB client version"); } - /** - * Get the internal API key used by YouTube website on InnerTube requests. - */ - public static String getKey() throws IOException, ExtractionException { - if (!isNullOrEmpty(key)) { - return key; - } - - // Always extract the key used by the website, by trying first to extract it from the - // JavaScript service worker, then from HTML search results page as a fallback, to prevent - // fingerprinting based on the key and/or invalid key issues - try { - extractClientVersionAndKeyFromSwJs(); - } catch (final Exception e) { - extractClientVersionAndKeyFromHtmlSearchResultsPage(); - } - - if (keyAndVersionExtracted) { - return key; - } - - // Fallback to the hardcoded one if it's valid - if (areHardcodedClientVersionAndKeyValid()) { - key = HARDCODED_KEY; - return key; - } - - // The ANDROID API key is also valid with the WEB client so return it if we couldn't - // extract the WEB API key. This can be used as a way to fingerprint the extractor in this - // case - return ANDROID_YOUTUBE_KEY; - } - /** *

* Only used in tests. @@ -742,10 +675,9 @@ public final class YoutubeParsingHelper { * tests with mocks will fail, because the mock is missing. *

*/ - public static void resetClientVersionAndKey() { + public static void resetClientVersion() { clientVersion = null; - key = null; - keyAndVersionExtracted = false; + clientVersionExtracted = false; } /** @@ -757,11 +689,11 @@ public final class YoutubeParsingHelper { numberGenerator = random; } - public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException, + public static boolean isHardcodedYoutubeMusicClientVersionValid() throws IOException, ReCaptchaException { final String url = - "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?key=" - + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER; + "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?" + + DISABLE_PRETTY_PRINT_PARAMETER; // @formatter:off final byte[] json = JsonWriter.string() @@ -769,7 +701,7 @@ public final class YoutubeParsingHelper { .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2]) + .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION) .value("hl", "en-GB") .value("gl", "GB") .value("platform", "DESKTOP") @@ -791,48 +723,39 @@ public final class YoutubeParsingHelper { // @formatter:on final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(HARDCODED_YOUTUBE_MUSIC_KEY[1], - HARDCODED_YOUTUBE_MUSIC_KEY[2])); + headers.putAll(getClientHeaders("67", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)); final Response response = getDownloader().postWithContentTypeJson(url, headers, json); // Ensure to have a valid response return response.responseBody().length() > 500 && response.responseCode() == 200; } - public static String[] getYoutubeMusicKey() + public static String getYoutubeMusicClientVersion() throws IOException, ReCaptchaException, Parser.RegexException { - if (youtubeMusicKey != null && youtubeMusicKey.length == 3) { - return youtubeMusicKey; + if (!isNullOrEmpty(youtubeMusicClientVersion)) { + return youtubeMusicClientVersion; } - if (isHardcodedYoutubeMusicKeyValid()) { - youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY; - return youtubeMusicKey; + if (isHardcodedYoutubeMusicClientVersionValid()) { + youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION; + return youtubeMusicClientVersion; } - String musicClientVersion; - String musicKey; - String musicClientName; - try { final String url = "https://music.youtube.com/sw.js"; final var headers = getOriginReferrerHeaders(YOUTUBE_MUSIC_URL); final String response = getDownloader().get(url, headers).responseBody(); - musicClientVersion = getStringResultFromRegexArray(response, + + youtubeMusicClientVersion = getStringResultFromRegexArray(response, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); - musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response); } catch (final Exception e) { final String url = "https://music.youtube.com/?ucbcb=1"; final String html = getDownloader().get(url, getCookieHeader()).responseBody(); - musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); - musicClientVersion = getStringResultFromRegexArray(html, + youtubeMusicClientVersion = getStringResultFromRegexArray(html, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html); } - youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion}; - return youtubeMusicKey; + return youtubeMusicClientVersion; } @Nullable @@ -1150,8 +1073,8 @@ public final class YoutubeParsingHelper { final var headers = getYouTubeHeaders(); return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key=" - + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); + getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?" + + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } public static JsonObject getJsonAndroidPostResponse( @@ -1160,7 +1083,7 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), ANDROID_YOUTUBE_KEY, endPartOfUrlRequest); + getAndroidUserAgent(localization), endPartOfUrlRequest); } public static JsonObject getJsonIosPostResponse( @@ -1169,7 +1092,7 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization), - IOS_YOUTUBE_KEY, endPartOfUrlRequest); + endPartOfUrlRequest); } private static JsonObject getMobilePostResponse( @@ -1177,12 +1100,11 @@ public final class YoutubeParsingHelper { final byte[] body, @Nonnull final Localization localization, @Nonnull final String userAgent, - @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final var headers = Map.of("User-Agent", List.of(userAgent), "X-Goog-Api-Format-Version", List.of("2")); - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" + DISABLE_PRETTY_PRINT_PARAMETER; return JsonUtils.toJsonObject(getValidJsonResponseBody( @@ -1302,7 +1224,7 @@ public final class YoutubeParsingHelper { The build version corresponding to the iOS version used can be found on https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15 */ - .value("osVersion", "17.1.2.21B101") + .value("osVersion", "17.4.1.21E237") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .value("utcOffsetMinutes", 0) @@ -1421,9 +1343,9 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 running iOS 17.1.2 with the hardcoded version of the iOS app + // Spoofing an iPhone 15 running iOS 17.4.1 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION - + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 17_1_2 like Mac OS X; " + + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 17_4_1 like Mac OS X; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ")"; } @@ -1434,7 +1356,7 @@ public final class YoutubeParsingHelper { @Nonnull public static Map> getYoutubeMusicHeaders() { final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(youtubeMusicKey[1], youtubeMusicKey[2])); + headers.putAll(getClientHeaders("67", youtubeMusicClientVersion)); return headers; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 49049457a..d2ec32ca6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -29,7 +29,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -422,8 +421,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); + return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null, + channelIds, null, body); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 29695366d..1df90f967 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; @@ -103,8 +102,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final var headers = getYouTubeHeaders(); final Response response = getDownloader().postWithContentTypeJson( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, - headers, body, localization); + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, + localization); initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); playlistData = initialData @@ -225,8 +224,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, - null, null, cookies, body); + return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, + cookies, body); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index dd2309448..4ed5449fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; @@ -25,10 +26,8 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; @@ -52,10 +51,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - - final String url = "https://music.youtube.com/youtubei/v1/search?key=" - + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = "https://music.youtube.com/youtubei/v1/search?" + + DISABLE_PRETTY_PRINT_PARAMETER; final String params; @@ -86,7 +83,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) + .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") @@ -206,15 +203,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - // @formatter:off final byte[] json = JsonWriter.string() .object() .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) + .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") @@ -295,8 +290,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } @Nullable - private Page getNextPageFrom(final JsonArray continuations) - throws IOException, ParsingException, ReCaptchaException { + private Page getNextPageFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { return null; } @@ -306,7 +300,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final String continuation = nextContinuationData.getString("continuation"); return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation - + "&continuation=" + continuation + "&key=" - + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER); + + "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index a8999fdd3..79afd5542 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -387,8 +386,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, body); + return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body); } else { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 90d5cab01..491d53164 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL; @@ -247,8 +246,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Nullable - private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, - ExtractionException { + private Page getNextPageFrom(final JsonObject continuationItemRenderer) { if (isNullOrEmpty(continuationItemRenderer)) { return null; } @@ -257,8 +255,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("continuationCommand") .getString("token"); - final String url = YOUTUBEI_V1_URL + "search?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER; return new Page(url, token); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java index 3974c3117..d172d8a8a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.Creati import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator; import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.AudioTrackType; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; import org.w3c.dom.Document; @@ -89,7 +88,7 @@ class YoutubeDashManifestCreatorsTest { @BeforeAll public static void setUp() throws Exception { - YoutubeParsingHelper.resetClientVersionAndKey(); + YoutubeParsingHelper.resetClientVersion(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); NewPipe.init(DownloaderTestImpl.getInstance()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 00e4a75ca..1a6e596d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import com.grack.nanojson.JsonWriter; @@ -91,7 +90,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -180,7 +179,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -268,7 +267,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -386,7 +385,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -453,7 +452,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -542,7 +541,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 905b3a01e..dc7733344 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -25,15 +25,15 @@ public class YoutubeParsingHelperTest { } @Test - void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException { - assertTrue(YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid(), - "Hardcoded client version and key are not valid anymore"); + void testIsHardcodedClientVersionValid() throws IOException, ExtractionException { + assertTrue(YoutubeParsingHelper.isHardcodedClientVersionValid(), + "Hardcoded client version is not valid anymore"); } @Test - void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException { - assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicKeyValid(), - "Hardcoded YouTube Music keys are not valid anymore"); + void testIsHardcodedYoutubeMusicClientVersionValid() throws IOException, ExtractionException { + assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicClientVersionValid(), + "Hardcoded YouTube Music client version is not valid anymore"); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index ca26eaf35..68d496415 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -26,7 +26,7 @@ public final class YoutubeTestsUtils { */ public static void ensureStateless() { YoutubeParsingHelper.setConsentAccepted(false); - YoutubeParsingHelper.resetClientVersionAndKey(); + YoutubeParsingHelper.resetClientVersion(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); YoutubeJavaScriptPlayerManager.clearAllCaches(); }