be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Improve payloads and URLs of InnerTube requests 

For every InnerTube request:
- Always add a `request` object with the following properties:
  - "internalExperimentFlags" set to an empty array;
  - "useSsl" set to "true";
  - "lockedSafetyMode" set to "false".
- Use proper TODO comment to provide a way to enable restricted mode on every
request and add it on requests on which it wasn't present.

For YouTube Music:
- Remove alt query parameter, as it is not used anymore by the website;
- Add prettyPrint query parameter with false value on YouTube Music search
continuations.
This commit is contained in:
AudricV 2023-10-07 15:00:40 +02:00
parent 8a9ebcc373
commit d97c9e0db1
No known key found for this signature in database
GPG Key ID: DA92EC7905614198
2 changed files with 59 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -551,12 +551,20 @@ public final class YoutubeParsingHelper {
.value("gl", "GB") .value("gl", "GB")
.value("clientName", "WEB") .value("clientName", "WEB")
.value("clientVersion", HARDCODED_CLIENT_VERSION) .value("clientVersion", HARDCODED_CLIENT_VERSION)
.value("platform", "DESKTOP")
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end() .end()
.object("user") .object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false) .value("lockedSafetyMode", false)
.end() .end()
.value("fetchLiveState", true)
.end() .end()
.value("fetchLiveState", true)
.end().done().getBytes(StandardCharsets.UTF_8); .end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on // @formatter:on
@ -770,7 +778,7 @@ public final class YoutubeParsingHelper {
public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException, public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
ReCaptchaException { ReCaptchaException {
final String url = final String url =
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key=" "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?key="
+ HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER; + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
// @formatter:off // @formatter:off
@ -782,19 +790,17 @@ public final class YoutubeParsingHelper {
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2]) .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2])
.value("hl", "en-GB") .value("hl", "en-GB")
.value("gl", "GB") .value("gl", "GB")
.array("experimentIds").end() .value("platform", "DESKTOP")
.value("experimentsToken", "")
.object("locationInfo").end()
.object("musicAppInfo").end()
.end() .end()
.object("capabilities").end()
.object("request") .object("request")
.array("internalExperimentFlags").end() .array("internalExperimentFlags")
.object("sessionIndex").end() .end()
.value("useSsl", true)
.end() .end()
.object("activePlayers").end()
.object("user") .object("user")
.value("enableSafetyMode", false) // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end() .end()
.end() .end()
.value("input", "") .value("input", "")
@ -1309,7 +1315,7 @@ public final class YoutubeParsingHelper {
.value("useSsl", true) .value("useSsl", true)
.end() .end()
.object("user") .object("user")
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean) // .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false) .value("lockedSafetyMode", false)
.end() .end()
@ -1345,8 +1351,13 @@ public final class YoutubeParsingHelper {
.value("hl", localization.getLocalizationCode()) .value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode()) .value("gl", contentCountry.getCountryCode())
.end() .end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user") .object("user")
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean) // .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false) .value("lockedSafetyMode", false)
.end() .end()
@ -1380,8 +1391,13 @@ public final class YoutubeParsingHelper {
.value("hl", localization.getLocalizationCode()) .value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode()) .value("gl", contentCountry.getCountryCode())
.end() .end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user") .object("user")
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean) // .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false) .value("lockedSafetyMode", false)
.end() .end()
@ -1408,8 +1424,13 @@ public final class YoutubeParsingHelper {
.object("thirdParty") .object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end() .end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user") .object("user")
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean) // .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false) .value("lockedSafetyMode", false)
.end() .end()

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -54,7 +54,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
throws IOException, ExtractionException { throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" final String url = "https://music.youtube.com/youtubei/v1/search?key="
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER; + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
final String params; final String params;
@ -89,20 +89,17 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("clientVersion", youtubeMusicKeys[2]) .value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en-GB") .value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode()) .value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end() .value("platform", "DESKTOP")
.value("experimentsToken", "")
.object("locationInfo").end()
.object("musicAppInfo").end()
.end() .end()
.object("capabilities").end()
.object("request") .object("request")
.array("internalExperimentFlags").end() .array("internalExperimentFlags")
.object("sessionIndex").end() .end()
.value("useSsl", true)
.end() .end()
.object("activePlayers").end()
.object("user") .object("user")
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
.value("enableSafetyMode", false) // .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end() .end()
.end() .end()
.value("query", getSearchString()) .value("query", getSearchString())
@ -219,20 +216,18 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("clientVersion", youtubeMusicKeys[2]) .value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en-GB") .value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode()) .value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end() .value("platform", "DESKTOP")
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0) .value("utcOffsetMinutes", 0)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end() .end()
.object("capabilities").end()
.object("request") .object("request")
.array("internalExperimentFlags").end() .array("internalExperimentFlags")
.object("sessionIndex").end() .end()
.value("useSsl", true)
.end() .end()
.object("activePlayers").end()
.object("user") .object("user")
.value("enableSafetyMode", false) // TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end() .end()
.end() .end()
.end().done().getBytes(StandardCharsets.UTF_8); .end().done().getBytes(StandardCharsets.UTF_8);
@ -310,7 +305,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String continuation = nextContinuationData.getString("continuation"); final String continuation = nextContinuationData.getString("continuation");
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
+ "&continuation=" + continuation + "&alt=json" + "&key=" + "&continuation=" + continuation + "&key="
+ YoutubeParsingHelper.getYoutubeMusicKey()[0]); + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
} }
} }