From d97c9e0db168c2cc12f0b216cb2c1eff2fad2196 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 7 Oct 2023 15:00:40 +0200 Subject: [PATCH] [YouTube] Improve payloads and URLs of InnerTube requests For every InnerTube request: - Always add a `request` object with the following properties: - "internalExperimentFlags" set to an empty array; - "useSsl" set to "true"; - "lockedSafetyMode" set to "false". - Use proper TODO comment to provide a way to enable restricted mode on every request and add it on requests on which it wasn't present. For YouTube Music: - Remove alt query parameter, as it is not used anymore by the website; - Add prettyPrint query parameter with false value on YouTube Music search continuations. --- .../youtube/YoutubeParsingHelper.java | 63 ++++++++++++------- .../YoutubeMusicSearchExtractor.java | 39 +++++------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index d1df41a0a..bb9d87ca3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -551,12 +551,20 @@ public final class YoutubeParsingHelper { .value("gl", "GB") .value("clientName", "WEB") .value("clientVersion", HARDCODED_CLIENT_VERSION) + .value("platform", "DESKTOP") + .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) .end() - .object("user") - .value("lockedSafetyMode", false) .end() .value("fetchLiveState", true) - .end() .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on @@ -770,7 +778,7 @@ public final class YoutubeParsingHelper { public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException, ReCaptchaException { final String url = - "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key=" + "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?key=" + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER; // @formatter:off @@ -782,19 +790,17 @@ public final class YoutubeParsingHelper { .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2]) .value("hl", "en-GB") .value("gl", "GB") - .array("experimentIds").end() - .value("experimentsToken", "") - .object("locationInfo").end() - .object("musicAppInfo").end() + .value("platform", "DESKTOP") .end() - .object("capabilities").end() .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() + .array("internalExperimentFlags") + .end() + .value("useSsl", true) .end() - .object("activePlayers").end() .object("user") - .value("enableSafetyMode", false) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) .end() .end() .value("input", "") @@ -1309,8 +1315,8 @@ public final class YoutubeParsingHelper { .value("useSsl", true) .end() .object("user") - // TO DO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end(); @@ -1345,9 +1351,14 @@ public final class YoutubeParsingHelper { .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() .object("user") - // TO DO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end(); @@ -1380,9 +1391,14 @@ public final class YoutubeParsingHelper { .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() .object("user") - // TO DO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end(); @@ -1408,9 +1424,14 @@ public final class YoutubeParsingHelper { .object("thirdParty") .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() .object("user") - // TO DO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 3efe6a74e..e7c108492 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -54,7 +54,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { throws IOException, ExtractionException { final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + final String url = "https://music.youtube.com/youtubei/v1/search?key=" + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER; final String params; @@ -89,20 +89,17 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .value("clientVersion", youtubeMusicKeys[2]) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .object("locationInfo").end() - .object("musicAppInfo").end() + .value("platform", "DESKTOP") .end() - .object("capabilities").end() .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() + .array("internalExperimentFlags") + .end() + .value("useSsl", true) .end() - .object("activePlayers").end() .object("user") - // TO DO: provide a way to enable restricted mode with: - .value("enableSafetyMode", false) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) .end() .end() .value("query", getSearchString()) @@ -219,20 +216,18 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .value("clientVersion", youtubeMusicKeys[2]) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") + .value("platform", "DESKTOP") .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() .end() - .object("capabilities").end() .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() + .array("internalExperimentFlags") + .end() + .value("useSsl", true) .end() - .object("activePlayers").end() .object("user") - .value("enableSafetyMode", false) + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) .end() .end() .end().done().getBytes(StandardCharsets.UTF_8); @@ -310,7 +305,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final String continuation = nextContinuationData.getString("continuation"); return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation - + "&continuation=" + continuation + "&alt=json" + "&key=" - + YoutubeParsingHelper.getYoutubeMusicKey()[0]); + + "&continuation=" + continuation + "&key=" + + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER); } }