From a86a30103fc4afc2bc199b57dc59d2fc9b7fe21b Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:24:56 +0200 Subject: [PATCH 01/13] [Youtube] bare bones version to solve throttling Done by transforming the parameter "n" from videoplayback urls https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-865985377 --- .../extractors/YoutubeStreamExtractor.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ad2b1f7db..03be11d35 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -39,6 +39,7 @@ import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.regex.Pattern; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; @@ -527,9 +528,31 @@ public class YoutubeStreamExtractor extends StreamExtractor { final List videoStreams = new ArrayList<>(); try { + getDeobfuscationCode(); + final String playerCode = NewPipe.getDownloader() + .get(playerJsUrl, getExtractorLocalization()).responseBody(); + Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); + String functionName = Parser.matchGroup1(pattern, playerCode); + Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); + String function = "function " + functionName + Parser.matchGroup1(functionPattern, playerCode); + + Context context = Context.enter(); + context.setOptimizationLevel(-1); + ScriptableObject scope = context.initSafeStandardObjects(); + for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { final ItagItem itag = entry.getValue(); - final VideoStream videoStream = new VideoStream(entry.getKey(), false, itag); + final String url = entry.getKey(); + Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); + String nValue = Parser.matchGroup1(nValuePattern, url); + + context.evaluateString(scope, function, functionName, 1, null); + final Function jsFunction = (Function) scope.get(functionName, scope); + Object result = jsFunction.call(context, scope, scope, new Object[]{nValue}); + String newNValue = Objects.toString(result, nValue); + String newUrl = nValuePattern.matcher(url).replaceFirst(newNValue); + System.out.println("aaaaaa " + nValue + " - " + newNValue); + final VideoStream videoStream = new VideoStream(newUrl, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); } From 80cf8b3acdc66efaa76f8bd67d1d6bf7a20b98f7 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Mon, 12 Jul 2021 20:06:19 +0200 Subject: [PATCH 02/13] Extract separate YoutubeThrottlingDecoder --- .../youtube/YoutubeThrottlingDecoder.java | 101 ++++++++++++++++++ .../extractors/YoutubeStreamExtractor.java | 77 +++++-------- .../newpipe/extractor/utils/Javascript.java | 24 +++++ 3 files changed, 151 insertions(+), 51 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java new file mode 100644 index 000000000..975df002a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java @@ -0,0 +1,101 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.Javascript; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.util.regex.Pattern; + +public class YoutubeThrottlingDecoder { + + private static final String HTTPS = "https:"; + + private final String functionName; + private final String function; + + public YoutubeThrottlingDecoder(String videoId, Localization localization) throws ParsingException { + String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl(videoId, localization)); + String playerJsCode = downloadPlayerJsCode(localization, playerJsUrl); + + functionName = parseDecodeFunctionName(playerJsCode); + function = parseDecodeFunction(playerJsCode, functionName); + } + + private String extractPlayerJsUrl(String videoId, Localization localization) throws ParsingException { + try { + final String embedUrl = "https://www.youtube.com/embed/" + videoId; + final String embedPageContent = NewPipe.getDownloader() + .get(embedUrl, localization).responseBody(); + + try { + final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; + return Parser.matchGroup1(assetsPattern, embedPageContent) + .replace("\\", "").replace("\"", ""); + } catch (final Parser.RegexException ex) { + // playerJsUrl is still available in the file, just somewhere else TODO + // it is ok not to find it, see how that's handled in getDeobfuscationCode() + final Document doc = Jsoup.parse(embedPageContent); + final Elements elems = doc.select("script").attr("name", "player_ias/base"); + for (final Element elem : elems) { + if (elem.attr("src").contains("base.js")) { + return elem.attr("src"); + } + } + } + + } catch (final Exception i) { + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + + private String cleanPlayerJsUrl(String playerJsUrl) { + if (playerJsUrl.startsWith("//")) { + return HTTPS + playerJsUrl; + } else if (playerJsUrl.startsWith("/")) { + // sometimes https://www.youtube.com part has to be added manually + return HTTPS + "//www.youtube.com" + playerJsUrl; + } else { + return playerJsUrl; + } + } + + private String downloadPlayerJsCode(Localization localization, String playerJsUrl) throws ParsingException { + try { + return NewPipe.getDownloader().get(playerJsUrl, localization).responseBody(); + } catch (Exception e) { + throw new ParsingException("Could not get player js code from url: " + playerJsUrl); + } + } + + private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException { + Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); + return Parser.matchGroup1(pattern, playerJsCode); + } + + private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException { + Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); + return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); + } + + public String parseNParam(String url) throws Parser.RegexException { + Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); + return Parser.matchGroup1(nValuePattern, url); + } + + public String decodeNParam(String nParam) { + Javascript javascript = new Javascript(); + return javascript.run(function, functionName, nParam); + } + + public String replaceNParam(String url, String newValue) { + Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); + return nValuePattern.matcher(url).replaceFirst(newValue); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 03be11d35..55ba96a41 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -25,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecoder; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -39,7 +40,6 @@ import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.regex.Pattern; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; @@ -80,13 +80,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private static String cachedDeobfuscationCode = null; - @Nullable - private String playerJsUrl = null; - - private JsonArray initialAjaxJson; - private JsonObject initialData; @Nonnull private final Map videoInfoPage = new HashMap<>(); + private JsonArray initialAjaxJson; + private JsonObject initialData; private JsonObject playerResponse; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; @@ -526,32 +523,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoStreams() throws ExtractionException { assertPageFetched(); final List videoStreams = new ArrayList<>(); + YoutubeThrottlingDecoder throttlingDecoder = new YoutubeThrottlingDecoder(getId(), getExtractorLocalization()); try { - getDeobfuscationCode(); - final String playerCode = NewPipe.getDownloader() - .get(playerJsUrl, getExtractorLocalization()).responseBody(); - Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); - String functionName = Parser.matchGroup1(pattern, playerCode); - Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); - String function = "function " + functionName + Parser.matchGroup1(functionPattern, playerCode); - - Context context = Context.enter(); - context.setOptimizationLevel(-1); - ScriptableObject scope = context.initSafeStandardObjects(); - for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { final ItagItem itag = entry.getValue(); final String url = entry.getKey(); - Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); - String nValue = Parser.matchGroup1(nValuePattern, url); - context.evaluateString(scope, function, functionName, 1, null); - final Function jsFunction = (Function) scope.get(functionName, scope); - Object result = jsFunction.call(context, scope, scope, new Object[]{nValue}); - String newNValue = Objects.toString(result, nValue); - String newUrl = nValuePattern.matcher(url).replaceFirst(newNValue); - System.out.println("aaaaaa " + nValue + " - " + newNValue); + String oldNParam = throttlingDecoder.parseNParam(url); + String newNParam = throttlingDecoder.decodeNParam(oldNParam); + String newUrl = throttlingDecoder.replaceNParam(url, newNParam); + + System.out.println("aaaaaa " + oldNParam + " - " + newNParam); final VideoStream videoStream = new VideoStream(newUrl, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); @@ -820,8 +803,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - @Nonnull - private String getEmbeddedInfoStsAndStorePlayerJsUrl() { + private String extractPlayerJsUrl() throws ParsingException { try { final String embedUrl = "https://www.youtube.com/embed/" + getId(); final String embedPageContent = NewPipe.getDownloader() @@ -829,7 +811,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent) + return Parser.matchGroup1(assetsPattern, embedPageContent) .replace("\\", "").replace("\"", ""); } catch (final Parser.RegexException ex) { // playerJsUrl is still available in the file, just somewhere else TODO @@ -838,17 +820,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Elements elems = doc.select("script").attr("name", "player_ias/base"); for (final Element elem : elems) { if (elem.attr("src").contains("base.js")) { - playerJsUrl = elem.attr("src"); - break; + return elem.attr("src"); } } } - // Get embed sts - return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent); } catch (final Exception i) { - // if it fails we simply reply with no sts as then it does not seem to be necessary - return ""; + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + + private String cleanPlayerJsUrl(String playerJsUrl) { + if (playerJsUrl.startsWith("//")) { + return HTTPS + playerJsUrl; + } else if (playerJsUrl.startsWith("/")) { + // sometimes https://www.youtube.com part has to be added manually + return HTTPS + "//www.youtube.com" + playerJsUrl; + } else { + return playerJsUrl; } } @@ -899,22 +889,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private String getDeobfuscationCode() throws ParsingException { if (cachedDeobfuscationCode == null) { - if (playerJsUrl == null) { - // the currentPlayerJsUrl was not found in any page fetched so far and there is - // nothing cached, so try fetching embedded info - getEmbeddedInfoStsAndStorePlayerJsUrl(); - if (playerJsUrl == null) { - throw new ParsingException( - "Embedded info did not provide YouTube player js url"); - } - } - - if (playerJsUrl.startsWith("//")) { - playerJsUrl = HTTPS + playerJsUrl; - } else if (playerJsUrl.startsWith("/")) { - // sometimes https://www.youtube.com part has to be added manually - playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl; - } + String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl()); cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java new file mode 100644 index 000000000..c7c81bcb5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.utils; + +import org.mozilla.javascript.Context; +import org.mozilla.javascript.Function; +import org.mozilla.javascript.ScriptableObject; + +public class Javascript { + + public String run(String function, String functionName, String... parameters) { + try { + Context context = Context.enter(); + context.setOptimizationLevel(-1); + ScriptableObject scope = context.initSafeStandardObjects(); + + context.evaluateString(scope, function, functionName, 1, null); + Function jsFunction = (Function) scope.get(functionName, scope); + Object result = jsFunction.call(context, scope, scope, parameters); + return result.toString(); + } finally { + Context.exit(); + } + } + +} From fcdb9bdbebc6271f7b7b9de7620ddd71d9af334f Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Thu, 15 Jul 2021 20:05:06 +0200 Subject: [PATCH 03/13] add url to sout --- .../extractor/services/youtube/YoutubeThrottlingDecoder.java | 5 +++-- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java index 975df002a..2a429d4e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java @@ -15,6 +15,7 @@ import java.util.regex.Pattern; public class YoutubeThrottlingDecoder { private static final String HTTPS = "https:"; + private static final String N_PARAM_REGEX = "[&?]n=([^&]+)"; private final String functionName; private final String function; @@ -85,7 +86,7 @@ public class YoutubeThrottlingDecoder { } public String parseNParam(String url) throws Parser.RegexException { - Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); + Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); return Parser.matchGroup1(nValuePattern, url); } @@ -95,7 +96,7 @@ public class YoutubeThrottlingDecoder { } public String replaceNParam(String url, String newValue) { - Pattern nValuePattern = Pattern.compile("[&?]n=([^&]+)"); + Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); return nValuePattern.matcher(url).replaceFirst(newValue); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 55ba96a41..27d5607eb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -534,7 +534,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String newNParam = throttlingDecoder.decodeNParam(oldNParam); String newUrl = throttlingDecoder.replaceNParam(url, newNParam); - System.out.println("aaaaaa " + oldNParam + " - " + newNParam); + System.out.println("aaaaaa " + url + " - " + oldNParam + " - " + newNParam); final VideoStream videoStream = new VideoStream(newUrl, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); From a189f685dcfc8d9bfb78fc71732c435c71ef0902 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Fri, 16 Jul 2021 19:20:40 +0200 Subject: [PATCH 04/13] cache player js code --- .../services/youtube/YoutubeThrottlingDecoder.java | 7 +++++-- .../youtube/extractors/YoutubeStreamExtractor.java | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java index 2a429d4e6..10bd2b43b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java @@ -16,13 +16,16 @@ public class YoutubeThrottlingDecoder { private static final String HTTPS = "https:"; private static final String N_PARAM_REGEX = "[&?]n=([^&]+)"; + private static String playerJsCode; private final String functionName; private final String function; public YoutubeThrottlingDecoder(String videoId, Localization localization) throws ParsingException { - String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl(videoId, localization)); - String playerJsCode = downloadPlayerJsCode(localization, playerJsUrl); + if (playerJsCode == null) { + String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl(videoId, localization)); + playerJsCode = downloadPlayerJsCode(localization, playerJsUrl); + } functionName = parseDecodeFunctionName(playerJsCode); function = parseDecodeFunction(playerJsCode, functionName); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 27d5607eb..1d2401075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -534,7 +534,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { String newNParam = throttlingDecoder.decodeNParam(oldNParam); String newUrl = throttlingDecoder.replaceNParam(url, newNParam); - System.out.println("aaaaaa " + url + " - " + oldNParam + " - " + newNParam); final VideoStream videoStream = new VideoStream(newUrl, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); From 6956b72af7967ea60a50f3c49a3ff1de8ff60f35 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 17 Jul 2021 10:05:14 +0200 Subject: [PATCH 05/13] Fix YoutubeThrottlingDecoder.replaceNParam Previously it replaced the parameter itself not the value of the parameter. --- .../youtube/YoutubeThrottlingDecoder.java | 5 +-- .../extractors/YoutubeStreamExtractor.java | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java index 10bd2b43b..05b871ce3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java @@ -98,8 +98,7 @@ public class YoutubeThrottlingDecoder { return javascript.run(function, functionName, nParam); } - public String replaceNParam(String url, String newValue) { - Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); - return nValuePattern.matcher(url).replaceFirst(newValue); + public String replaceNParam(String url, String oldValue, String newValue) { + return url.replace(oldValue, newValue); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 1d2401075..25a37cec7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -17,7 +18,15 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.*; +import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; +import org.schabi.newpipe.extractor.exceptions.PaidContentException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.PrivateContentException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; @@ -27,21 +36,42 @@ import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecoder; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamSegment; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -532,7 +562,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String oldNParam = throttlingDecoder.parseNParam(url); String newNParam = throttlingDecoder.decodeNParam(oldNParam); - String newUrl = throttlingDecoder.replaceNParam(url, newNParam); + String newUrl = throttlingDecoder.replaceNParam(url, oldNParam, newNParam); final VideoStream videoStream = new VideoStream(newUrl, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { From a02ee2e952a4d002491c07de4296d0abd3401f72 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 17 Jul 2021 19:10:09 +0200 Subject: [PATCH 06/13] Rewrite youtube throttling solution and add tests --- .../youtube/YoutubeJavascriptExtractor.java | 101 +++++++++++++++++ .../youtube/YoutubeThrottlingDecoder.java | 104 ------------------ .../youtube/YoutubeThrottlingDecrypter.java | 88 +++++++++++++++ .../extractors/YoutubeStreamExtractor.java | 55 ++------- .../YoutubeJavascriptExtractorTest.java | 47 ++++++++ .../YoutubeThrottlingDecrypterTest.java | 40 +++++++ 6 files changed, 287 insertions(+), 148 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java new file mode 100644 index 000000000..06f2c44c2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java @@ -0,0 +1,101 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.Parser; + +import javax.annotation.Nonnull; + +/** + * Youtube restricts streaming their media in multiple ways by requiring clients to apply a cipher function + * on parameters of requests. + * The cipher function is sent alongside as a JavaScript function. + *

+ * This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions. + */ +public class YoutubeJavascriptExtractor { + + private static final String HTTPS = "https:"; + private static String cachedJavascriptCode; + + /** + * Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls. + * + * @param videoId Does not influence the result, but a valid video id can prevent tracking + * @return The whole javascript file as a string. + * @throws ParsingException If the extraction failed. + */ + @Nonnull + public static String extractJavascriptCode(String videoId) throws ParsingException { + if (cachedJavascriptCode == null) { + final YoutubeJavascriptExtractor extractor = new YoutubeJavascriptExtractor(); + String playerJsUrl = extractor.cleanJavascriptUrl(extractor.extractJavascriptUrl(videoId)); + cachedJavascriptCode = extractor.downloadJavascriptCode(playerJsUrl); + } + + return cachedJavascriptCode; + } + + /** + * Same as {@link YoutubeJavascriptExtractor#extractJavascriptCode(String)} but with a constant value for videoId. + * Possible because the videoId has no influence on the result. + * + * For tracking avoidance purposes it may make sense to pass in valid video ids. + */ + @Nonnull + public static String extractJavascriptCode() throws ParsingException { + return extractJavascriptCode("d4IGg5dqeO8"); + } + + private String extractJavascriptUrl(String videoId) throws ParsingException { + try { + final String embedUrl = "https://www.youtube.com/embed/" + videoId; + final String embedPageContent = NewPipe.getDownloader() + .get(embedUrl, Localization.DEFAULT).responseBody(); + + try { + final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; + return Parser.matchGroup1(assetsPattern, embedPageContent) + .replace("\\", "").replace("\"", ""); + } catch (final Parser.RegexException ex) { + // playerJsUrl is still available in the file, just somewhere else TODO + // it is ok not to find it, see how that's handled in getDeobfuscationCode() + final Document doc = Jsoup.parse(embedPageContent); + final Elements elems = doc.select("script").attr("name", "player_ias/base"); + for (final Element elem : elems) { + if (elem.attr("src").contains("base.js")) { + return elem.attr("src"); + } + } + } + + } catch (final Exception i) { + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + throw new ParsingException("Embedded info did not provide YouTube player js url"); + } + + private String cleanJavascriptUrl(String playerJsUrl) { + if (playerJsUrl.startsWith("//")) { + return HTTPS + playerJsUrl; + } else if (playerJsUrl.startsWith("/")) { + // sometimes https://www.youtube.com part has to be added manually + return HTTPS + "//www.youtube.com" + playerJsUrl; + } else { + return playerJsUrl; + } + } + + private String downloadJavascriptCode(String playerJsUrl) throws ParsingException { + try { + return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody(); + } catch (Exception e) { + throw new ParsingException("Could not get player js code from url: " + playerJsUrl); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java deleted file mode 100644 index 05b871ce3..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecoder.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.utils.Javascript; -import org.schabi.newpipe.extractor.utils.Parser; - -import java.util.regex.Pattern; - -public class YoutubeThrottlingDecoder { - - private static final String HTTPS = "https:"; - private static final String N_PARAM_REGEX = "[&?]n=([^&]+)"; - private static String playerJsCode; - - private final String functionName; - private final String function; - - public YoutubeThrottlingDecoder(String videoId, Localization localization) throws ParsingException { - if (playerJsCode == null) { - String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl(videoId, localization)); - playerJsCode = downloadPlayerJsCode(localization, playerJsUrl); - } - - functionName = parseDecodeFunctionName(playerJsCode); - function = parseDecodeFunction(playerJsCode, functionName); - } - - private String extractPlayerJsUrl(String videoId, Localization localization) throws ParsingException { - try { - final String embedUrl = "https://www.youtube.com/embed/" + videoId; - final String embedPageContent = NewPipe.getDownloader() - .get(embedUrl, localization).responseBody(); - - try { - final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - return Parser.matchGroup1(assetsPattern, embedPageContent) - .replace("\\", "").replace("\"", ""); - } catch (final Parser.RegexException ex) { - // playerJsUrl is still available in the file, just somewhere else TODO - // it is ok not to find it, see how that's handled in getDeobfuscationCode() - final Document doc = Jsoup.parse(embedPageContent); - final Elements elems = doc.select("script").attr("name", "player_ias/base"); - for (final Element elem : elems) { - if (elem.attr("src").contains("base.js")) { - return elem.attr("src"); - } - } - } - - } catch (final Exception i) { - throw new ParsingException("Embedded info did not provide YouTube player js url"); - } - throw new ParsingException("Embedded info did not provide YouTube player js url"); - } - - private String cleanPlayerJsUrl(String playerJsUrl) { - if (playerJsUrl.startsWith("//")) { - return HTTPS + playerJsUrl; - } else if (playerJsUrl.startsWith("/")) { - // sometimes https://www.youtube.com part has to be added manually - return HTTPS + "//www.youtube.com" + playerJsUrl; - } else { - return playerJsUrl; - } - } - - private String downloadPlayerJsCode(Localization localization, String playerJsUrl) throws ParsingException { - try { - return NewPipe.getDownloader().get(playerJsUrl, localization).responseBody(); - } catch (Exception e) { - throw new ParsingException("Could not get player js code from url: " + playerJsUrl); - } - } - - private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException { - Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); - return Parser.matchGroup1(pattern, playerJsCode); - } - - private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException { - Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); - return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); - } - - public String parseNParam(String url) throws Parser.RegexException { - Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); - return Parser.matchGroup1(nValuePattern, url); - } - - public String decodeNParam(String nParam) { - Javascript javascript = new Javascript(); - return javascript.run(function, functionName, nParam); - } - - public String replaceNParam(String url, String oldValue, String newValue) { - return url.replace(oldValue, newValue); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java new file mode 100644 index 000000000..d8295113d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Javascript; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.util.regex.Pattern; + +/** + *

+ * YouTube's media is protected with a cipher, which modifies the "n" query parameter of it's video playback urls. + * This class handles extracting that "n" query parameter, applying the cipher on it and returning the resulting url + * which is not throttled. + *

+ * + *

+ * https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&other=other + *

+ * becomes + *

+ * https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other + *

+ */ +public class YoutubeThrottlingDecrypter { + + private static final String N_PARAM_REGEX = "[&?]n=([^&]+)"; + + private final String functionName; + private final String function; + + /** + *

+ * Use this if you care about the off chance that YouTube tracks with which videoId the cipher is requested. + *

+ * Otherwise use the no-arg constructor which uses a constant value. + */ + public YoutubeThrottlingDecrypter(String videoId) throws ParsingException { + final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(videoId); + + functionName = parseDecodeFunctionName(playerJsCode); + function = parseDecodeFunction(playerJsCode, functionName); + } + + public YoutubeThrottlingDecrypter() throws ParsingException { + final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(); + + functionName = parseDecodeFunctionName(playerJsCode); + function = parseDecodeFunction(playerJsCode, functionName); + } + + private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException { + Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); + return Parser.matchGroup1(pattern, playerJsCode); + } + + private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException { + Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); + return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); + } + + public String apply(String url) throws Parser.RegexException { + if (containsNParam(url)) { + String oldNParam = parseNParam(url); + String newNParam = decryptNParam(oldNParam); + return replaceNParam(url, oldNParam, newNParam); + } else { + return url; + } + } + + private boolean containsNParam(String url) { + return Parser.isMatch(N_PARAM_REGEX, url); + } + + private String parseNParam(String url) throws Parser.RegexException { + Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); + return Parser.matchGroup1(nValuePattern, url); + } + + private String decryptNParam(String nParam) { + Javascript javascript = new Javascript(); + return javascript.run(function, functionName, nParam); + } + + private String replaceNParam(String url, String oldValue, String newValue) { + return url.replace(oldValue, newValue); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 25a37cec7..bfb765278 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -18,15 +17,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; -import org.schabi.newpipe.extractor.exceptions.PaidContentException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.PrivateContentException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; +import org.schabi.newpipe.extractor.exceptions.*; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; @@ -34,44 +25,23 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecoder; +import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.Frameset; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamSegment; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; +import java.util.*; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -553,18 +523,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoStreams() throws ExtractionException { assertPageFetched(); final List videoStreams = new ArrayList<>(); - YoutubeThrottlingDecoder throttlingDecoder = new YoutubeThrottlingDecoder(getId(), getExtractorLocalization()); + YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { final ItagItem itag = entry.getValue(); - final String url = entry.getKey(); + String url = entry.getKey(); + url = throttlingDecrypter.apply(url); - String oldNParam = throttlingDecoder.parseNParam(url); - String newNParam = throttlingDecoder.decodeNParam(oldNParam); - String newUrl = throttlingDecoder.replaceNParam(url, oldNParam, newNParam); - - final VideoStream videoStream = new VideoStream(newUrl, false, itag); + final VideoStream videoStream = new VideoStream(url, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java new file mode 100644 index 000000000..c626f55da --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; + +public class YoutubeJavascriptExtractorTest { + + @Before + public void setup() throws IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testExtractJavascript__success() throws ParsingException { + String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("d4IGg5dqeO8"); + assertPlayerJsCode(playerJsCode); + + playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(); + assertPlayerJsCode(playerJsCode); + } + + @Test + public void testExtractJavascript__invalidVideoId__success() throws ParsingException { + String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("not_a_video_id"); + assertPlayerJsCode(playerJsCode); + + playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("11-chars123"); + assertPlayerJsCode(playerJsCode); + + } + + private void assertPlayerJsCode(String playerJsCode) { + assertThat(playerJsCode, allOf( + containsString(" Copyright The Closure Library Authors.\n" + + " SPDX-License-Identifier: Apache-2.0"), + containsString("var _yt_player"))); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java new file mode 100644 index 000000000..74ed90a48 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java @@ -0,0 +1,40 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +public class YoutubeThrottlingDecrypterTest { + + @Before + public void setup() throws IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testDecode__success() throws ParsingException { + // urls extracted from browser with the dev tools + final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; + final String decryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=HI_QWjzacNoiJw&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; + + final String actualDecryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl); + + assertThat(actualDecryptedUrl, equalTo(decryptedUrl)); + } + + @Test + public void testDecode__noNParam__success() throws ParsingException { + final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw=="; + String decrypted = new YoutubeThrottlingDecrypter().apply(noNParamUrl); + + assertThat(decrypted, equalTo(noNParamUrl)); + } + +} \ No newline at end of file From a683c8d2784984f0d93f330a824046e1b9c8f081 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 17 Jul 2021 19:14:57 +0200 Subject: [PATCH 07/13] Delete duplicated code to load youtube javascript file --- .../extractors/YoutubeStreamExtractor.java | 51 ++----------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bfb765278..a0fa228a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -24,6 +24,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavascriptExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -799,45 +800,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private String extractPlayerJsUrl() throws ParsingException { - try { - final String embedUrl = "https://www.youtube.com/embed/" + getId(); - final String embedPageContent = NewPipe.getDownloader() - .get(embedUrl, getExtractorLocalization()).responseBody(); - - try { - final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - return Parser.matchGroup1(assetsPattern, embedPageContent) - .replace("\\", "").replace("\"", ""); - } catch (final Parser.RegexException ex) { - // playerJsUrl is still available in the file, just somewhere else TODO - // it is ok not to find it, see how that's handled in getDeobfuscationCode() - final Document doc = Jsoup.parse(embedPageContent); - final Elements elems = doc.select("script").attr("name", "player_ias/base"); - for (final Element elem : elems) { - if (elem.attr("src").contains("base.js")) { - return elem.attr("src"); - } - } - } - - } catch (final Exception i) { - throw new ParsingException("Embedded info did not provide YouTube player js url"); - } - throw new ParsingException("Embedded info did not provide YouTube player js url"); - } - - private String cleanPlayerJsUrl(String playerJsUrl) { - if (playerJsUrl.startsWith("//")) { - return HTTPS + playerJsUrl; - } else if (playerJsUrl.startsWith("/")) { - // sometimes https://www.youtube.com part has to be added manually - return HTTPS + "//www.youtube.com" + playerJsUrl; - } else { - return playerJsUrl; - } - } - private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { Parser.RegexException exception = null; for (final String regex : REGEXES) { @@ -852,11 +814,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); } - private String loadDeobfuscationCode(@Nonnull final String playerJsUrl) + private String loadDeobfuscationCode() throws DeobfuscateException { try { - final String playerCode = NewPipe.getDownloader() - .get(playerJsUrl, getExtractorLocalization()).responseBody(); + final String playerCode = YoutubeJavascriptExtractor.extractJavascriptCode(getId()); final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); final String functionPattern = "(" @@ -875,8 +836,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}"; return helperObject + deobfuscateFunction + callerFunction; - } catch (final IOException ioe) { - throw new DeobfuscateException("Could not load deobfuscate function", ioe); } catch (final Exception e) { throw new DeobfuscateException("Could not parse deobfuscate function ", e); } @@ -885,9 +844,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private String getDeobfuscationCode() throws ParsingException { if (cachedDeobfuscationCode == null) { - String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl()); - - cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl); + cachedDeobfuscationCode = loadDeobfuscationCode(); } return cachedDeobfuscationCode; } From 3a3d1d7f2bd364cdb5e81a6a1171471ca55f1fc2 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 20 Jul 2021 20:48:11 +0200 Subject: [PATCH 08/13] Make YoutubeJavaScriptExtractor and JavaScript methods static Also address review and rewrite some comments --- ...r.java => YoutubeJavaScriptExtractor.java} | 40 ++++++++++--------- .../youtube/YoutubeThrottlingDecrypter.java | 9 ++--- .../extractors/YoutubeStreamExtractor.java | 10 ++--- .../{Javascript.java => JavaScript.java} | 7 +++- ...va => YoutubeJavaScriptExtractorTest.java} | 14 +++---- 5 files changed, 41 insertions(+), 39 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{YoutubeJavascriptExtractor.java => YoutubeJavaScriptExtractor.java} (72%) rename extractor/src/main/java/org/schabi/newpipe/extractor/utils/{Javascript.java => JavaScript.java} (82%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{YoutubeJavascriptExtractorTest.java => YoutubeJavaScriptExtractorTest.java} (69%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java similarity index 72% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java index 06f2c44c2..9cfcfa3e0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java @@ -12,47 +12,51 @@ import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; /** - * Youtube restricts streaming their media in multiple ways by requiring clients to apply a cipher function + * YouTube restricts streaming their media in multiple ways by requiring clients to apply a cipher function * on parameters of requests. * The cipher function is sent alongside as a JavaScript function. *

* This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions. */ -public class YoutubeJavascriptExtractor { +public class YoutubeJavaScriptExtractor { private static final String HTTPS = "https:"; - private static String cachedJavascriptCode; + private static String cachedJavaScriptCode; + + private YoutubeJavaScriptExtractor() { + } /** * Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls. * - * @param videoId Does not influence the result, but a valid video id can prevent tracking + * @param videoId Does not influence the result, but a valid video id may help in the chance that YouTube tracks it. * @return The whole javascript file as a string. * @throws ParsingException If the extraction failed. */ @Nonnull - public static String extractJavascriptCode(String videoId) throws ParsingException { - if (cachedJavascriptCode == null) { - final YoutubeJavascriptExtractor extractor = new YoutubeJavascriptExtractor(); - String playerJsUrl = extractor.cleanJavascriptUrl(extractor.extractJavascriptUrl(videoId)); - cachedJavascriptCode = extractor.downloadJavascriptCode(playerJsUrl); + public static String extractJavaScriptCode(String videoId) throws ParsingException { + if (cachedJavaScriptCode == null) { + final String playerJsUrl = YoutubeJavaScriptExtractor.cleanJavaScriptUrl( + YoutubeJavaScriptExtractor.extractJavaScriptUrl(videoId)); + cachedJavaScriptCode = YoutubeJavaScriptExtractor.downloadJavaScriptCode(playerJsUrl); } - return cachedJavascriptCode; + return cachedJavaScriptCode; } /** - * Same as {@link YoutubeJavascriptExtractor#extractJavascriptCode(String)} but with a constant value for videoId. + * Same as {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)} but with a constant value for videoId. * Possible because the videoId has no influence on the result. - * - * For tracking avoidance purposes it may make sense to pass in valid video ids. + *

+ * In the off chance that YouTube tracks with which video id the request is made, it may make sense to pass in + * video ids. */ @Nonnull - public static String extractJavascriptCode() throws ParsingException { - return extractJavascriptCode("d4IGg5dqeO8"); + public static String extractJavaScriptCode() throws ParsingException { + return extractJavaScriptCode("d4IGg5dqeO8"); } - private String extractJavascriptUrl(String videoId) throws ParsingException { + private static String extractJavaScriptUrl(String videoId) throws ParsingException { try { final String embedUrl = "https://www.youtube.com/embed/" + videoId; final String embedPageContent = NewPipe.getDownloader() @@ -80,7 +84,7 @@ public class YoutubeJavascriptExtractor { throw new ParsingException("Embedded info did not provide YouTube player js url"); } - private String cleanJavascriptUrl(String playerJsUrl) { + private static String cleanJavaScriptUrl(String playerJsUrl) { if (playerJsUrl.startsWith("//")) { return HTTPS + playerJsUrl; } else if (playerJsUrl.startsWith("/")) { @@ -91,7 +95,7 @@ public class YoutubeJavascriptExtractor { } } - private String downloadJavascriptCode(String playerJsUrl) throws ParsingException { + private static String downloadJavaScriptCode(String playerJsUrl) throws ParsingException { try { return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody(); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index d8295113d..18cc872f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.utils.Javascript; +import org.schabi.newpipe.extractor.utils.JavaScript; import org.schabi.newpipe.extractor.utils.Parser; import java.util.regex.Pattern; @@ -35,14 +35,14 @@ public class YoutubeThrottlingDecrypter { * Otherwise use the no-arg constructor which uses a constant value. */ public YoutubeThrottlingDecrypter(String videoId) throws ParsingException { - final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(videoId); + final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); functionName = parseDecodeFunctionName(playerJsCode); function = parseDecodeFunction(playerJsCode, functionName); } public YoutubeThrottlingDecrypter() throws ParsingException { - final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(); + final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); functionName = parseDecodeFunctionName(playerJsCode); function = parseDecodeFunction(playerJsCode, functionName); @@ -78,8 +78,7 @@ public class YoutubeThrottlingDecrypter { } private String decryptNParam(String nParam) { - Javascript javascript = new Javascript(); - return javascript.run(function, functionName, nParam); + return JavaScript.run(function, functionName, nParam); } private String replaceNParam(String url, String oldValue, String newValue) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a0fa228a9..0ad22d349 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,10 +4,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -24,7 +20,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.services.youtube.YoutubeJavascriptExtractor; +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -524,7 +520,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoStreams() throws ExtractionException { assertPageFetched(); final List videoStreams = new ArrayList<>(); - YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); + final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { @@ -817,7 +813,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String loadDeobfuscationCode() throws DeobfuscateException { try { - final String playerCode = YoutubeJavascriptExtractor.extractJavascriptCode(getId()); + final String playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(getId()); final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); final String functionPattern = "(" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java similarity index 82% rename from extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java index c7c81bcb5..a25b3d4ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Javascript.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java @@ -4,9 +4,12 @@ import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -public class Javascript { +public class JavaScript { - public String run(String function, String functionName, String... parameters) { + private JavaScript() { + } + + public static String run(String function, String functionName, String... parameters) { try { Context context = Context.enter(); context.setOptimizationLevel(-1); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java similarity index 69% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java index c626f55da..45ebf9092 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavascriptExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java @@ -12,7 +12,7 @@ import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; -public class YoutubeJavascriptExtractorTest { +public class YoutubeJavaScriptExtractorTest { @Before public void setup() throws IOException { @@ -20,20 +20,20 @@ public class YoutubeJavascriptExtractorTest { } @Test - public void testExtractJavascript__success() throws ParsingException { - String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("d4IGg5dqeO8"); + public void testExtractJavaScript__success() throws ParsingException { + String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("d4IGg5dqeO8"); assertPlayerJsCode(playerJsCode); - playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(); + playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); assertPlayerJsCode(playerJsCode); } @Test - public void testExtractJavascript__invalidVideoId__success() throws ParsingException { - String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("not_a_video_id"); + public void testExtractJavaScript__invalidVideoId__success() throws ParsingException { + String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("not_a_video_id"); assertPlayerJsCode(playerJsCode); - playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("11-chars123"); + playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("11-chars123"); assertPlayerJsCode(playerJsCode); } From 1c30a2725e242b418194f0e2943868ae5a692ea7 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Thu, 22 Jul 2021 18:52:58 +0200 Subject: [PATCH 09/13] Cache nParams to prevent executing the JavaScript function for the same nParam multiple times. Closes #689 --- .../services/youtube/YoutubeThrottlingDecrypter.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index 18cc872f3..28cc9c37b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -4,6 +4,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.JavaScript; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Pattern; /** @@ -24,6 +26,7 @@ import java.util.regex.Pattern; public class YoutubeThrottlingDecrypter { private static final String N_PARAM_REGEX = "[&?]n=([^&]+)"; + private static final Map nParams = new HashMap<>(); private final String functionName; private final String function; @@ -78,7 +81,12 @@ public class YoutubeThrottlingDecrypter { } private String decryptNParam(String nParam) { - return JavaScript.run(function, functionName, nParam); + if (nParams.containsKey(nParam)) { + return nParams.get(nParam); + } + final String decryptedNParam = JavaScript.run(function, functionName, nParam); + nParams.put(nParam, decryptedNParam); + return decryptedNParam; } private String replaceNParam(String url, String oldValue, String newValue) { From 32055147e04d70d2e62b7314cec6ff3a67b72855 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Tue, 27 Jul 2021 17:24:27 +0200 Subject: [PATCH 10/13] Do some code improvements Use final where possible, annotate some methods and parameters as Nonnull and format new code to be in the 100 characters limit per line. --- .../youtube/YoutubeJavaScriptExtractor.java | 35 +++++++++++-------- .../youtube/YoutubeThrottlingDecrypter.java | 34 +++++++++++------- .../newpipe/extractor/utils/JavaScript.java | 12 ++++--- .../YoutubeJavaScriptExtractorTest.java | 2 +- 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java index 9cfcfa3e0..7b8d83dd6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java @@ -12,11 +12,12 @@ import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; /** - * YouTube restricts streaming their media in multiple ways by requiring clients to apply a cipher function - * on parameters of requests. + * YouTube restricts streaming their media in multiple ways by requiring clients to apply a cipher + * function on parameters of requests. * The cipher function is sent alongside as a JavaScript function. *

- * This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions. + * This class handling fetching the JavaScript file in order to allow other classes to extract the + * needed functions. */ public class YoutubeJavaScriptExtractor { @@ -27,14 +28,16 @@ public class YoutubeJavaScriptExtractor { } /** - * Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls. + * Extracts the JavaScript file. The result is cached, so subsequent calls use the result of + * previous calls. * - * @param videoId Does not influence the result, but a valid video id may help in the chance that YouTube tracks it. - * @return The whole javascript file as a string. + * @param videoId Does not influence the result, but a valid video id may help in the chance + * that YouTube tracks it. + * @return The whole JavaScript file as a string. * @throws ParsingException If the extraction failed. */ @Nonnull - public static String extractJavaScriptCode(String videoId) throws ParsingException { + public static String extractJavaScriptCode(final String videoId) throws ParsingException { if (cachedJavaScriptCode == null) { final String playerJsUrl = YoutubeJavaScriptExtractor.cleanJavaScriptUrl( YoutubeJavaScriptExtractor.extractJavaScriptUrl(videoId)); @@ -45,18 +48,19 @@ public class YoutubeJavaScriptExtractor { } /** - * Same as {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)} but with a constant value for videoId. + * Same as {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)} but with a constant + * value for videoId. * Possible because the videoId has no influence on the result. *

- * In the off chance that YouTube tracks with which video id the request is made, it may make sense to pass in - * video ids. + * In the off chance that YouTube tracks with which video id the request is made, it may make + * sense to pass in video ids. */ @Nonnull public static String extractJavaScriptCode() throws ParsingException { return extractJavaScriptCode("d4IGg5dqeO8"); } - private static String extractJavaScriptUrl(String videoId) throws ParsingException { + private static String extractJavaScriptUrl(final String videoId) throws ParsingException { try { final String embedUrl = "https://www.youtube.com/embed/" + videoId; final String embedPageContent = NewPipe.getDownloader() @@ -84,7 +88,8 @@ public class YoutubeJavaScriptExtractor { throw new ParsingException("Embedded info did not provide YouTube player js url"); } - private static String cleanJavaScriptUrl(String playerJsUrl) { + @Nonnull + private static String cleanJavaScriptUrl(@Nonnull final String playerJsUrl) { if (playerJsUrl.startsWith("//")) { return HTTPS + playerJsUrl; } else if (playerJsUrl.startsWith("/")) { @@ -95,10 +100,12 @@ public class YoutubeJavaScriptExtractor { } } - private static String downloadJavaScriptCode(String playerJsUrl) throws ParsingException { + @Nonnull + private static String downloadJavaScriptCode(final String playerJsUrl) + throws ParsingException { try { return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody(); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get player js code from url: " + playerJsUrl); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index 28cc9c37b..48ddab82e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.JavaScript; import org.schabi.newpipe.extractor.utils.Parser; +import javax.annotation.Nonnull; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; @@ -33,11 +34,12 @@ public class YoutubeThrottlingDecrypter { /** *

- * Use this if you care about the off chance that YouTube tracks with which videoId the cipher is requested. + * Use this if you care about the off chance that YouTube tracks with which videoId the cipher + * is requested. *

* Otherwise use the no-arg constructor which uses a constant value. */ - public YoutubeThrottlingDecrypter(String videoId) throws ParsingException { + public YoutubeThrottlingDecrypter(final String videoId) throws ParsingException { final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); functionName = parseDecodeFunctionName(playerJsCode); @@ -51,17 +53,22 @@ public class YoutubeThrottlingDecrypter { function = parseDecodeFunction(playerJsCode, functionName); } - private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException { - Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); + private String parseDecodeFunctionName(final String playerJsCode) + throws Parser.RegexException { + Pattern pattern = Pattern.compile( + "b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)"); return Parser.matchGroup1(pattern, playerJsCode); } - private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException { - Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL); - return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); + @Nonnull + private String parseDecodeFunction(final String playerJsCode, final String functionName) + throws Parser.RegexException { + Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", + Pattern.DOTALL); + return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); } - public String apply(String url) throws Parser.RegexException { + public String apply(final String url) throws Parser.RegexException { if (containsNParam(url)) { String oldNParam = parseNParam(url); String newNParam = decryptNParam(oldNParam); @@ -71,16 +78,16 @@ public class YoutubeThrottlingDecrypter { } } - private boolean containsNParam(String url) { + private boolean containsNParam(final String url) { return Parser.isMatch(N_PARAM_REGEX, url); } - private String parseNParam(String url) throws Parser.RegexException { + private String parseNParam(final String url) throws Parser.RegexException { Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX); return Parser.matchGroup1(nValuePattern, url); } - private String decryptNParam(String nParam) { + private String decryptNParam(final String nParam) { if (nParams.containsKey(nParam)) { return nParams.get(nParam); } @@ -89,7 +96,10 @@ public class YoutubeThrottlingDecrypter { return decryptedNParam; } - private String replaceNParam(String url, String oldValue, String newValue) { + @Nonnull + private String replaceNParam(@Nonnull final String url, + final String oldValue, + final String newValue) { return url.replace(oldValue, newValue); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java index a25b3d4ab..4b81ba7d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java @@ -9,15 +9,17 @@ public class JavaScript { private JavaScript() { } - public static String run(String function, String functionName, String... parameters) { + public static String run(final String function, + final String functionName, + final String... parameters) { try { - Context context = Context.enter(); + final Context context = Context.enter(); context.setOptimizationLevel(-1); - ScriptableObject scope = context.initSafeStandardObjects(); + final ScriptableObject scope = context.initSafeStandardObjects(); context.evaluateString(scope, function, functionName, 1, null); - Function jsFunction = (Function) scope.get(functionName, scope); - Object result = jsFunction.call(context, scope, scope, parameters); + final Function jsFunction = (Function) scope.get(functionName, scope); + final Object result = jsFunction.call(context, scope, scope, parameters); return result.toString(); } finally { Context.exit(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java index 45ebf9092..f86dbf716 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java @@ -38,7 +38,7 @@ public class YoutubeJavaScriptExtractorTest { } - private void assertPlayerJsCode(String playerJsCode) { + private void assertPlayerJsCode(final String playerJsCode) { assertThat(playerJsCode, allOf( containsString(" Copyright The Closure Library Authors.\n" + " SPDX-License-Identifier: Apache-2.0"), From d13f531b6ff985b5ca51166e0c59ef3dc4a87b59 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Tue, 27 Jul 2021 17:28:32 +0200 Subject: [PATCH 11/13] Use YoutubeThrottlingDecrypter also in getAudioStreams and getVideoOnlyStreams methods of YoutubeStreamExtractor Without this commit, the n param is only decrypted for streams extracted in getVideoStreams (so only for streams in the formats object of the player response). --- .../youtube/extractors/YoutubeStreamExtractor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0ad22d349..403ae16cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -500,11 +500,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getAudioStreams() throws ExtractionException { assertPageFetched(); final List audioStreams = new ArrayList<>(); + final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { final ItagItem itag = entry.getValue(); - final AudioStream audioStream = new AudioStream(entry.getKey(), itag); + String url = entry.getKey(); + url = throttlingDecrypter.apply(url); + + final AudioStream audioStream = new AudioStream(url, itag); if (!Stream.containSimilarStream(audioStream, audioStreams)) { audioStreams.add(audioStream); } @@ -544,11 +548,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); final List videoOnlyStreams = new ArrayList<>(); + final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); + try { for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { final ItagItem itag = entry.getValue(); + String url = entry.getKey(); + url = throttlingDecrypter.apply(url); - final VideoStream videoStream = new VideoStream(entry.getKey(), true, itag); + final VideoStream videoStream = new VideoStream(url, true, itag); if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) { videoOnlyStreams.add(videoStream); } From d70adfdb8f5fc1e69d36328463e27118efbdb3a6 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 27 Jul 2021 20:31:20 +0200 Subject: [PATCH 12/13] Add methods for cache control to YoutubeThrottlingDecrypter. --- .../youtube/YoutubeThrottlingDecrypter.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index 48ddab82e..4f10e8c58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -11,9 +11,10 @@ import java.util.regex.Pattern; /** *

- * YouTube's media is protected with a cipher, which modifies the "n" query parameter of it's video playback urls. - * This class handles extracting that "n" query parameter, applying the cipher on it and returning the resulting url - * which is not throttled. + * YouTube's media is protected with a cipher, + * which modifies the "n" query parameter of it's video playback urls. + * This class handles extracting that "n" query parameter, + * applying the cipher on it and returning the resulting url which is not throttled. *

* *

@@ -23,6 +24,12 @@ import java.util.regex.Pattern; *

* https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other *

+ *
+ *

+ * Decoding the "n" parameter is time intensive. For this reason, the results are cached. + * The cache can be cleared using {@link #clearCache()} + *

+ * */ public class YoutubeThrottlingDecrypter { @@ -102,4 +109,18 @@ public class YoutubeThrottlingDecrypter { final String newValue) { return url.replace(oldValue, newValue); } + + /** + * @return the number of the cached "n" query parameters. + */ + public static int getCacheSize() { + return nParams.size(); + } + + /** + * Clears all stored "n" query parameters. + */ + public static void clearCache() { + nParams.clear(); + } } From 79f2d74b041d3749971dfcd7db0710f15e53920f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 28 Jul 2021 16:45:31 +0200 Subject: [PATCH 13/13] Fix test The decryption code changes over time. Only check whether the n parameter was changed in the URL --- .../youtube/YoutubeThrottlingDecrypterTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java index 74ed90a48..f072615f6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNotEquals; public class YoutubeThrottlingDecrypterTest { @@ -20,13 +21,11 @@ public class YoutubeThrottlingDecrypterTest { @Test public void testDecode__success() throws ParsingException { - // urls extracted from browser with the dev tools + // URL extracted from browser with the dev tools final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; - final String decryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=HI_QWjzacNoiJw&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; - - final String actualDecryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl); - - assertThat(actualDecryptedUrl, equalTo(decryptedUrl)); + final String decryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl); + // The cipher function changes over time, so we just check if the n param changed. + assertNotEquals(encryptedUrl, decryptedUrl); } @Test