- * YouTube restrict streaming their media in multiple ways by requiring their HTML5 clients to use - * a signature timestamp, and on streaming URLs a signature deobfuscation function for some - * contents and a throttling parameter deobfuscation one for all contents. - *
- * - ** This class handles fetching of this base JavaScript player file in order to allow other classes * to extract the needed data. *
@@ -31,7 +27,7 @@ import java.util.regex.Pattern; * watch page as a fallback. * */ -public final class YoutubeJavaScriptExtractor { +final class YoutubeJavaScriptExtractor { private static final String HTTPS = "https:"; private static final String BASE_JS_PLAYER_URL_FORMAT = @@ -40,49 +36,45 @@ public final class YoutubeJavaScriptExtractor { "player\\\\/([a-z0-9]{8})\\\\/"); private static final Pattern EMBEDDED_WATCH_PAGE_JS_BASE_PLAYER_URL_PATTERN = Pattern.compile( "\"jsUrl\":\"(/s/player/[A-Za-z0-9]+/player_ias\\.vflset/[A-Za-z_-]+/base\\.js)\""); - private static String cachedJavaScriptCode; private YoutubeJavaScriptExtractor() { } /** - * Extracts the JavaScript file. + * Extracts the JavaScript base player file. * - *- * The result is cached, so subsequent calls use the result of previous calls. - *
- * - * @param videoId a YouTube video ID, which doesn't influence the result, but it may help in - * the chance that YouTube track it - * @return the whole JavaScript file as a string - * @throws ParsingException if the extraction failed + * @param videoId the video ID used to get the JavaScript base player file (an empty one can be + * passed, even it is not recommend in order to spoof better official YouTube + * clients) + * @return the whole JavaScript base player file as a string + * @throws ParsingException if the extraction of the file failed */ @Nonnull - public static String extractJavaScriptCode(@Nonnull final String videoId) + static String extractJavaScriptPlayerCode(@Nonnull final String videoId) throws ParsingException { - if (cachedJavaScriptCode == null) { - String url; - try { - url = YoutubeJavaScriptExtractor.extractJavaScriptUrlWithIframeResource(); - } catch (final Exception e) { - url = YoutubeJavaScriptExtractor.extractJavaScriptUrlWithEmbedWatchPage(videoId); - } + String url; + try { + url = YoutubeJavaScriptExtractor.extractJavaScriptUrlWithIframeResource(); final String playerJsUrl = YoutubeJavaScriptExtractor.cleanJavaScriptUrl(url); - cachedJavaScriptCode = YoutubeJavaScriptExtractor.downloadJavaScriptCode(playerJsUrl); + + // Assert that the URL we extracted and built is valid + new URL(playerJsUrl); + + return YoutubeJavaScriptExtractor.downloadJavaScriptCode(playerJsUrl); + } catch (final Exception e) { + url = YoutubeJavaScriptExtractor.extractJavaScriptUrlWithEmbedWatchPage(videoId); + final String playerJsUrl = YoutubeJavaScriptExtractor.cleanJavaScriptUrl(url); + + try { + // Assert that the URL we extracted and built is valid + new URL(playerJsUrl); + } catch (final MalformedURLException exception) { + throw new ParsingException( + "The extracted and built JavaScript URL is invalid", exception); + } + + return YoutubeJavaScriptExtractor.downloadJavaScriptCode(playerJsUrl); } - - return cachedJavaScriptCode; - } - - /** - * Reset the cached JavaScript code. - * - *- * It will be fetched again the next time {@link #extractJavaScriptCode(String)} is called. - *
- */ - public static void resetJavaScriptCode() { - cachedJavaScriptCode = null; } @Nonnull @@ -134,7 +126,7 @@ public final class YoutubeJavaScriptExtractor { } } - // Use regexes to match the URL in a JavaScript embedded script of the HTML page + // Use regexes to match the URL in an embedded script of the HTML page try { return Parser.matchGroup1( EMBEDDED_WATCH_PAGE_JS_BASE_PLAYER_URL_PATTERN, embedPageContent); @@ -145,29 +137,28 @@ public final class YoutubeJavaScriptExtractor { } @Nonnull - private static String cleanJavaScriptUrl(@Nonnull final String playerJsUrl) { - if (playerJsUrl.startsWith("//")) { + private static String cleanJavaScriptUrl(@Nonnull final String javaScriptPlayerUrl) { + if (javaScriptPlayerUrl.startsWith("//")) { // https part has to be added manually if the URL is protocol-relative - return HTTPS + playerJsUrl; - } else if (playerJsUrl.startsWith("/")) { + return HTTPS + javaScriptPlayerUrl; + } else if (javaScriptPlayerUrl.startsWith("/")) { // https://www.youtube.com part has to be added manually if the URL is relative to // YouTube's domain - return HTTPS + "//www.youtube.com" + playerJsUrl; + return HTTPS + "//www.youtube.com" + javaScriptPlayerUrl; } else { - return playerJsUrl; + return javaScriptPlayerUrl; } } @Nonnull - private static String downloadJavaScriptCode(@Nonnull final String playerJsUrl) + private static String downloadJavaScriptCode(@Nonnull final String javaScriptPlayerUrl) throws ParsingException { try { return NewPipe.getDownloader() - .get(playerJsUrl, Localization.DEFAULT) + .get(javaScriptPlayerUrl, Localization.DEFAULT) .responseBody(); } catch (final Exception e) { - throw new ParsingException( - "Could not get JavaScript base player's code from URL: " + playerJsUrl, e); + throw new ParsingException("Could not get JavaScript base player's code", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptPlayerManager.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptPlayerManager.java new file mode 100644 index 000000000..7ddfdb193 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptPlayerManager.java @@ -0,0 +1,334 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.JavaScript; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Manage the extraction and the usage of YouTube's player JavaScript needed data in the YouTube + * service. + * + *+ * YouTube restrict streaming their media in multiple ways by requiring their HTML5 clients to use + * a signature timestamp, and on streaming URLs a signature deobfuscation function for some + * contents and a throttling parameter deobfuscation one for all contents. + *
+ * + *+ * This class provides access to methods which allows to get base JavaScript player's signature + * timestamp and to deobfuscate streaming URLs' signature and/or throttling parameter of HTML5 + * clients. + *
+ */ +public final class YoutubeJavaScriptPlayerManager { + + @Nonnull + private static final Map+ * A valid signature timestamp sent in the payload of player InnerTube requests is required to + * get valid stream URLs on HTML5 clients for videos which have obfuscated signatures. + *
+ * + *+ * The base JavaScript player file will fetched if it is not already done. + *
+ * + *+ * The result of the extraction is cached until {@link #clearAllCaches()} is called, making + * subsequent calls faster. + *
+ * + * @param videoId the video ID used to get the JavaScript base player file (an empty one can be + * passed, even it is not recommend in order to spoof better official YouTube + * clients) + * @return the signature timestamp of the base JavaScript player file + * @throws ParsingException if the extraction of the base JavaScript player file or the + * signature timestamp failed + */ + @Nonnull + public static String getSignatureTimestamp(@Nonnull final String videoId) + throws ParsingException { + // Return the cached result if it is present + if (cachedSignatureTimestamp != null) { + return cachedSignatureTimestamp; + } + + // If the signature timestamp has been not extracted on a previous call, this mean that we + // will fail to extract it on next calls too if the player code has been not changed + // Throw again the corresponding stored exception in this case to improve performance + if (sigTimestampExtractionEx != null) { + throw sigTimestampExtractionEx; + } + + extractJavaScriptCodeIfNeeded(videoId); + + try { + cachedSignatureTimestamp = YoutubeSignatureUtils.getSignatureTimestamp( + cachedJavaScriptPlayerCode); + } catch (final ParsingException e) { + // Store the exception for future calls of this method, in order to improve performance + sigTimestampExtractionEx = e; + throw e; + } + + return cachedSignatureTimestamp; + } + + /** + * Deobfuscate a signature of a streaming URL using its corresponding JavaScript base player's + * function. + * + *+ * Obfuscated signatures are only present on streaming URLs of some videos with HTML5 clients. + *
+ * + * @param videoId the video ID used to get the JavaScript base player file (an + * empty one can be passed, even it is not recommend in order to + * spoof better official YouTube clients) + * @param obfuscatedSignature the obfuscated signature of a streaming URL + * @return the deobfuscated signature + * @throws ParsingException if the extraction of the base JavaScript player file or the + * signature deobfuscation function failed + */ + @Nonnull + public static String deobfuscateSignature(@Nonnull final String videoId, + @Nonnull final String obfuscatedSignature) + throws ParsingException { + // If the signature deobfuscation function has been not extracted on a previous call, this + // mean that we will fail to extract it on next calls too if the player code has been not + // changed + // Throw again the corresponding stored exception in this case to improve performance + if (sigDeobFuncExtractionEx != null) { + throw sigDeobFuncExtractionEx; + } + + extractJavaScriptCodeIfNeeded(videoId); + + if (cachedSignatureDeobfuscationFunction == null) { + try { + cachedSignatureDeobfuscationFunction = YoutubeSignatureUtils.getDeobfuscationCode( + cachedJavaScriptPlayerCode); + } catch (final ParsingException e) { + // Store the exception for future calls of this method, in order to improve + // performance + sigDeobFuncExtractionEx = e; + throw e; + } + } + + try { + // Return an empty parameter in the case the function returns null + return Objects.requireNonNullElse( + JavaScript.run(cachedSignatureDeobfuscationFunction, + YoutubeSignatureUtils.DEOBFUSCATION_FUNCTION_NAME, + obfuscatedSignature), ""); + } catch (final Exception e) { + // This shouldn't happen as the function validity is checked when it is extracted + throw new ParsingException( + "Could not run signature parameter deobfuscation JavaScript function", e); + } + } + + /** + * Return a streaming URL with the throttling parameter of a given one deobfuscated, if it is + * present, using its corresponding JavaScript base player's function. + * + *+ * The throttling parameter is present on all streaming URLs of HTML5 clients. + *
+ * + *+ * If it is not given or deobfuscated, speeds will be throttled to a very slow speed (around 50 + * KB/s) and some streaming URLs could even lead to invalid HTTP responses such a 403 one. + *
+ * + *+ * As throttling parameters can be common between multiple streaming URLs of the same player + * response, deobfuscated parameters are cached with their obfuscated variant, in order to + * improve performance with multiple calls of this method having the same obfuscated throttling + * parameter. + *
+ * + *+ * The cache's size can be get using {@link #getThrottlingParametersCacheSize()} and the cache + * can be cleared using {@link #clearThrottlingParametersCache()} or {@link #clearAllCaches()}. + *
+ * + * @param videoId the video ID used to get the JavaScript base player file (an empty one + * can be passed, even it is not recommend in order to spoof better + * official YouTube clients) + * @param streamingUrl a streaming URL + * @return the original streaming URL if it has no throttling parameter or a URL with a + * deobfuscated throttling parameter + * @throws ParsingException if the extraction of the base JavaScript player file or the + * throttling parameter deobfuscation function failed + */ + @Nonnull + public static String getUrlWithThrottlingParameterDeobfuscated( + @Nonnull final String videoId, + @Nonnull final String streamingUrl) throws ParsingException { + final String obfuscatedThrottlingParameter = + YoutubeThrottlingParameterUtils.getThrottlingParameterFromStreamingUrl( + streamingUrl); + // If the throttling parameter is not present, return the original streaming URL + if (obfuscatedThrottlingParameter == null) { + return streamingUrl; + } + + // Do not use the containsKey method of the Map interface in order to avoid a double + // element search, and so to improve performance + final String cacheResult = CACHED_THROTTLING_PARAMETERS.get( + obfuscatedThrottlingParameter); + if (cacheResult != null) { + // If the throttling parameter function has been already ran on the throttling parameter + // of the current streaming URL, replace directly the obfuscated throttling parameter + // with the cached result in the streaming URL + return streamingUrl.replace(obfuscatedThrottlingParameter, cacheResult); + } + + extractJavaScriptCodeIfNeeded(videoId); + + // If the throttling parameter deobfuscation function has been not extracted on a previous + // call, this mean that we will fail to extract it on next calls too if the player code has + // been not changed + // Throw again the corresponding stored exception in this case to improve performance + if (throttlingDeobfFuncExtractionEx != null) { + throw throttlingDeobfFuncExtractionEx; + } + + if (cachedThrottlingDeobfuscationFunction == null) { + try { + cachedThrottlingDeobfuscationFunctionName = + YoutubeThrottlingParameterUtils.getDeobfuscationFunctionName( + cachedJavaScriptPlayerCode); + + cachedThrottlingDeobfuscationFunction = + YoutubeThrottlingParameterUtils.getDeobfuscationFunction( + cachedJavaScriptPlayerCode, + cachedThrottlingDeobfuscationFunctionName); + } catch (final ParsingException e) { + // Store the exception for future calls of this method, in order to improve + // performance + throttlingDeobfFuncExtractionEx = e; + throw e; + } + } + + try { + final String deobfuscatedThrottlingParameter = JavaScript.run( + cachedThrottlingDeobfuscationFunction, + cachedThrottlingDeobfuscationFunctionName, + obfuscatedThrottlingParameter); + + CACHED_THROTTLING_PARAMETERS.put( + obfuscatedThrottlingParameter, deobfuscatedThrottlingParameter); + + return streamingUrl.replace( + obfuscatedThrottlingParameter, deobfuscatedThrottlingParameter); + } catch (final Exception e) { + // This shouldn't happen as the function validity is checked when it is extracted + throw new ParsingException( + "Could not run throttling parameter deobfuscation JavaScript function", e); + } + } + + /** + * Get the current cache size of throttling parameters. + * + * @return the current cache size of throttling parameters + */ + public static int getThrottlingParametersCacheSize() { + return CACHED_THROTTLING_PARAMETERS.size(); + } + + /** + * Clear all caches. + * + *+ * This method will clear all cached JavaScript code and throttling parameters. + *
+ * + *+ * The next time {@link #getSignatureTimestamp(String)}, + * {@link #deobfuscateSignature(String, String)} or + * {@link #getUrlWithThrottlingParameterDeobfuscated(String, String)} is called, the JavaScript + * code will be fetched again and the corresponding extraction methods will be ran. + *
+ */ + public static void clearAllCaches() { + cachedJavaScriptPlayerCode = null; + cachedSignatureDeobfuscationFunction = null; + cachedThrottlingDeobfuscationFunctionName = null; + cachedThrottlingDeobfuscationFunction = null; + cachedSignatureTimestamp = null; + clearThrottlingParametersCache(); + + // Clear cached extraction exceptions, if applicable + throttlingDeobfFuncExtractionEx = null; + sigDeobFuncExtractionEx = null; + sigTimestampExtractionEx = null; + } + + /** + * Clear all cached throttling parameters. + * + *+ * The throttling parameter deobfuscation function will be ran again on these parameters if + * streaming URLs containing them are passed in the future. + *
+ * + *+ * This method doesn't clear the cached throttling parameter deobfuscation function, this can + * be done using {@link #clearAllCaches()}. + *
+ */ + public static void clearThrottlingParametersCache() { + CACHED_THROTTLING_PARAMETERS.clear(); + } + + /** + * Extract the JavaScript code if it isn't already cached. + * + * @param videoId the video ID used to get the JavaScript base player file (an empty one can be + * passed, even it is not recommend in order to spoof better official YouTube + * clients) + * @throws ParsingException if the extraction of the base JavaScript player file failed + */ + private static void extractJavaScriptCodeIfNeeded(@Nonnull final String videoId) + throws ParsingException { + if (cachedJavaScriptPlayerCode == null) { + cachedJavaScriptPlayerCode = YoutubeJavaScriptExtractor.extractJavaScriptPlayerCode( + videoId); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java deleted file mode 100644 index 227751e29..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.utils.JavaScript; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.jsextractor.JavaScriptExtractor; - -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.annotation.Nonnull; - -/** - * YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their - * {@code n} query parameter. - * - *- * This class handles extracting that {@code n} query parameter, applying the cipher on it and - * returning the resulting URL which is not throttled. - *
- * - *- * For instance, - * {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&other=other} - * becomes - * {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other}. - *
- * - *- * Decoding the {@code n} parameter is time intensive. For this reason, the results are cached. - * The cache can be cleared using {@link #clearCache()}. - *
- * - */ -public final class YoutubeThrottlingDecrypter { - - private static final Pattern N_PARAM_PATTERN = Pattern.compile("[&?]n=([^&]+)"); - private static final Pattern DECRYPT_FUNCTION_NAME_PATTERN = Pattern.compile( - // CHECKSTYLE:OFF - "\\.get\\(\"n\"\\)\\)&&\\([a-zA-Z0-9$_]=([a-zA-Z0-9$_]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9$_]\\)"); - // CHECKSTYLE:ON - - // Escape the curly end brace to allow compatibility with Android's regex engine - // See https://stackoverflow.com/q/45074813 - @SuppressWarnings("RegExpRedundantEscape") - private static final String DECRYPT_FUNCTION_BODY_REGEX = - "=\\s*function([\\S\\s]*?\\}\\s*return [\\w$]+?\\.join\\(\"\"\\)\\s*\\};)"; - - private static final String DECRYPT_FUNCTION_ARRAY_OBJECT_TYPE_DECLARATION_REGEX = "var "; - private static final String FUNCTION_NAMES_IN_DECRYPT_ARRAY_REGEX = "\\s*=\\s*\\[(.+?)][;,]"; - - private static final Map- * If the streaming URL provided doesn't contain a throttling parameter, it is returned as it - * is; otherwise, the encrypted value is decrypted and this value is replaced by the decrypted - * one. - *
- * - *- * If the JavaScript code has been not extracted, it is extracted with the given video ID using - * {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)}. - *
- * - * @param streamingUrl The streaming URL to decrypt, if needed. - * @param videoId A video ID, used to fetch the JavaScript code to get the decryption - * function. It can be a constant value of any existing video, but a - * constant value is discouraged, because it could allow tracking. - * @return A streaming URL with the decrypted parameter or the streaming URL itself if no - * throttling parameter has been found. - * @throws ParsingException If the streaming URL contains a throttling parameter and its - * decryption failed - */ - public static String apply(@Nonnull final String streamingUrl, - @Nonnull final String videoId) throws ParsingException { - if (!containsNParam(streamingUrl)) { - return streamingUrl; - } - - try { - if (decryptFunction == null) { - final String playerJsCode - = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); - - decryptFunctionName = parseDecodeFunctionName(playerJsCode); - decryptFunction = parseDecodeFunction(playerJsCode, decryptFunctionName); - } - - final String oldNParam = parseNParam(streamingUrl); - final String newNParam = decryptNParam(decryptFunction, decryptFunctionName, oldNParam); - return replaceNParam(streamingUrl, oldNParam, newNParam); - } catch (final Exception e) { - throw new ParsingException("Could not parse, decrypt or replace n parameter", e); - } - } - - private static String parseDecodeFunctionName(final String playerJsCode) - throws Parser.RegexException { - final Matcher matcher = DECRYPT_FUNCTION_NAME_PATTERN.matcher(playerJsCode); - if (!matcher.find()) { - throw new Parser.RegexException("Failed to find pattern \"" - + DECRYPT_FUNCTION_NAME_PATTERN + "\""); - } - - final String functionName = matcher.group(1); - if (matcher.groupCount() == 1) { - return functionName; - } - - final int arrayNum = Integer.parseInt(matcher.group(2)); - final Pattern arrayPattern = Pattern.compile( - DECRYPT_FUNCTION_ARRAY_OBJECT_TYPE_DECLARATION_REGEX + Pattern.quote(functionName) - + FUNCTION_NAMES_IN_DECRYPT_ARRAY_REGEX); - final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode); - final String[] names = arrayStr.split(","); - return names[arrayNum]; - } - - @Nonnull - private static String parseDecodeFunction(final String playerJsCode, final String functionName) - throws Parser.RegexException { - try { - return parseWithLexer(playerJsCode, functionName); - } catch (final Exception e) { - return parseWithRegex(playerJsCode, functionName); - } - } - - @Nonnull - private static String parseWithRegex(final String playerJsCode, final String functionName) - throws Parser.RegexException { - // Quote the function name, as it may contain special regex characters such as dollar - final Pattern functionPattern = Pattern.compile( - Pattern.quote(functionName) + DECRYPT_FUNCTION_BODY_REGEX, Pattern.DOTALL); - return validateFunction("function " - + functionName - + Parser.matchGroup1(functionPattern, playerJsCode)); - } - - @Nonnull - private static String validateFunction(@Nonnull final String function) { - JavaScript.compileOrThrow(function); - return function; - } - - @Nonnull - private static String parseWithLexer(final String playerJsCode, final String functionName) - throws ParsingException { - final String functionBase = functionName + "=function"; - return functionBase + JavaScriptExtractor.matchToClosingBrace(playerJsCode, functionBase) - + ";"; - } - - private static boolean containsNParam(final String url) { - return Parser.isMatch(N_PARAM_PATTERN, url); - } - - private static String parseNParam(final String url) throws Parser.RegexException { - return Parser.matchGroup1(N_PARAM_PATTERN, url); - } - - private static String decryptNParam(final String function, - final String functionName, - final String nParam) { - if (N_PARAMS_CACHE.containsKey(nParam)) { - return N_PARAMS_CACHE.get(nParam); - } - final String decryptedNParam = JavaScript.run(function, functionName, nParam); - N_PARAMS_CACHE.put(nParam, decryptedNParam); - return decryptedNParam; - } - - @Nonnull - private static String replaceNParam(@Nonnull final String url, - final String oldValue, - final String newValue) { - return url.replace(oldValue, newValue); - } - - /** - * @return The number of the cached {@code n} query parameters. - */ - public static int getCacheSize() { - return N_PARAMS_CACHE.size(); - } - - /** - * Clears all stored {@code n} query parameters. - */ - public static void clearCache() { - N_PARAMS_CACHE.clear(); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java new file mode 100644 index 000000000..c398c6202 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -0,0 +1,137 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.JavaScript; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.jsextractor.JavaScriptExtractor; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class to get the throttling parameter decryption code and check if a streaming has the + * throttling parameter. + */ +final class YoutubeThrottlingParameterUtils { + + private static final Pattern THROTTLING_PARAM_PATTERN = Pattern.compile("[&?]n=([^&]+)"); + + private static final Pattern DEOBFUSCATION_FUNCTION_NAME_PATTERN = Pattern.compile( + // CHECKSTYLE:OFF + "\\.get\\(\"n\"\\)\\)&&\\([a-zA-Z0-9$_]=([a-zA-Z0-9$_]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9$_]\\)"); + // CHECKSTYLE:ON + + // Escape the curly end brace to allow compatibility with Android's regex engine + // See https://stackoverflow.com/q/45074813 + @SuppressWarnings("RegExpRedundantEscape") + private static final String DEOBFUSCATION_FUNCTION_BODY_REGEX = + "=\\s*function([\\S\\s]*?\\}\\s*return [\\w$]+?\\.join\\(\"\"\\)\\s*\\};)"; + + private static final String DEOBFUSCATION_FUNCTION_ARRAY_OBJECT_TYPE_DECLARATION_REGEX = "var "; + + private static final String FUNCTION_NAMES_IN_DEOBFUSCATION_ARRAY_REGEX = + "\\s*=\\s*\\[(.+?)][;,]"; + + private YoutubeThrottlingParameterUtils() { + } + + /** + * Get the throttling parameter deobfuscation function name of YouTube's base JavaScript file. + * + * @param javaScriptPlayerCode the complete JavaScript base player code + * @return the name of the throttling parameter deobfuscation function + * @throws ParsingException if the name of the throttling parameter deobfuscation function + * could not be extracted + */ + @Nonnull + static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode) + throws ParsingException { + final Matcher matcher = DEOBFUSCATION_FUNCTION_NAME_PATTERN.matcher(javaScriptPlayerCode); + if (!matcher.find()) { + throw new ParsingException("Failed to find deobfuscation function name pattern \"" + + DEOBFUSCATION_FUNCTION_NAME_PATTERN + + "\" in the base JavaScript player code"); + } + + final String functionName = matcher.group(1); + if (matcher.groupCount() == 1) { + return functionName; + } + + final int arrayNum = Integer.parseInt(matcher.group(2)); + final Pattern arrayPattern = Pattern.compile( + DEOBFUSCATION_FUNCTION_ARRAY_OBJECT_TYPE_DECLARATION_REGEX + + Pattern.quote(functionName) + + FUNCTION_NAMES_IN_DEOBFUSCATION_ARRAY_REGEX); + final String arrayStr = Parser.matchGroup1(arrayPattern, javaScriptPlayerCode); + final String[] names = arrayStr.split(","); + return names[arrayNum]; + } + + /** + * Get the throttling parameter deobfuscation code of YouTube's base JavaScript file. + * + * @param javaScriptPlayerCode the complete JavaScript base player code + * @return the throttling parameter deobfuscation function name + * @throws ParsingException if the throttling parameter deobfuscation code couldn't be + * extracted + */ + @Nonnull + static String getDeobfuscationFunction(@Nonnull final String javaScriptPlayerCode, + @Nonnull final String functionName) + throws ParsingException { + try { + return parseFunctionWithLexer(javaScriptPlayerCode, functionName); + } catch (final Exception e) { + return parseFunctionWithRegex(javaScriptPlayerCode, functionName); + } + } + + /** + * Get the throttling parameter of a streaming URL if it exists. + * + * @param streamingUrl a streaming URL + * @return the throttling parameter of the streaming URL or {@code null} if no parameter has + * been found + */ + @Nullable + static String getThrottlingParameterFromStreamingUrl(@Nonnull final String streamingUrl) { + try { + return Parser.matchGroup1(THROTTLING_PARAM_PATTERN, streamingUrl); + } catch (final Parser.RegexException e) { + // If the throttling parameter could not be parsed from the URL, it means that there is + // no throttling parameter + // Return null in this case + return null; + } + } + + @Nonnull + private static String parseFunctionWithLexer(@Nonnull final String javaScriptPlayerCode, + @Nonnull final String functionName) + throws ParsingException { + final String functionBase = functionName + "=function"; + return functionBase + JavaScriptExtractor.matchToClosingBrace( + javaScriptPlayerCode, functionBase) + ";"; + } + + @Nonnull + private static String parseFunctionWithRegex(@Nonnull final String javaScriptPlayerCode, + @Nonnull final String functionName) + throws Parser.RegexException { + // Quote the function name, as it may contain special regex characters such as dollar + final Pattern functionPattern = Pattern.compile( + Pattern.quote(functionName) + DEOBFUSCATION_FUNCTION_BODY_REGEX, + Pattern.DOTALL); + return validateFunction("function " + functionName + + Parser.matchGroup1(functionPattern, javaScriptPlayerCode)); + } + + @Nonnull + private static String validateFunction(@Nonnull final String function) { + JavaScript.compileOrThrow(function); + return function; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c78ee5b96..c42349c50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -45,9 +45,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; -import org.mozilla.javascript.Context; -import org.mozilla.javascript.Function; -import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; @@ -69,9 +66,8 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor; +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; @@ -107,25 +103,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; public class YoutubeStreamExtractor extends StreamExtractor { - /*////////////////////////////////////////////////////////////////////////// - // Exceptions - //////////////////////////////////////////////////////////////////////////*/ - - public static class DeobfuscateException extends ParsingException { - DeobfuscateException(final String message, final Throwable cause) { - super(message, cause); - } - } - - /*////////////////////////////////////////////////////////////////////////*/ - - @Nullable - private static String cachedDeobfuscationCode = null; - @Nullable - private static String sts = null; - @Nullable - private static String playerCode = null; - private static boolean isAndroidClientFetchForced = false; private static boolean isIosClientFetchForced = false; @@ -637,19 +614,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Try to decrypt a streaming URL and fall back to the given URL, because decryption may fail - * if YouTube changes break something. + * Try to deobfuscate a streaming URL and fall back to the given URL, because decryption may + * fail if YouTube changes break something. * ** This way a breaking change from YouTube does not result in a broken extractor. *
* - * @param streamingUrl the streaming URL to decrypt with {@link YoutubeThrottlingDecrypter} + * @param streamingUrl the streaming URL to which deobfuscating its throttling parameter if + * there is one * @param videoId the video ID to use when extracting JavaScript player code, if needed */ - private String tryDecryptUrl(final String streamingUrl, final String videoId) { + private String tryDeobfuscateThrottlingParameterOfUrl(@Nonnull final String streamingUrl, + @Nonnull final String videoId) { try { - return YoutubeThrottlingDecrypter.apply(streamingUrl, videoId); + return YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( + videoId, streamingUrl); } catch (final ParsingException e) { return streamingUrl; } @@ -781,36 +761,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; - private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate"; private static final String STREAMING_DATA = "streamingData"; private static final String PLAYER = "player"; private static final String NEXT = "next"; private static final String SIGNATURE_CIPHER = "signatureCipher"; private static final String CIPHER = "cipher"; - private static final String[] REGEXES = { - "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)" - + "\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", - "\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)", - "\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)", - "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", - "\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", - "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" - }; - private static final String STS_REGEX = "signatureTimestamp[=:](\\d+)"; - @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final String videoId = getId(); - initStsFromPlayerJsIfNeeded(videoId); final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); html5Cpn = generateContentPlaybackNonce(); playerResponse = getJsonPostResponse(PLAYER, - createDesktopPlayerBody(localization, contentCountry, videoId, sts, false, + createDesktopPlayerBody( + localization, + contentCountry, + videoId, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId), + false, html5Cpn), localization); @@ -1044,7 +1016,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { html5Cpn = generateContentPlaybackNonce(); final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER, - createDesktopPlayerBody(localization, contentCountry, videoId, sts, true, + createDesktopPlayerBody(localization, + contentCountry, + videoId, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId), + true, html5Cpn), localization); if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) { @@ -1096,106 +1072,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("videoId")); } - private static void storePlayerJs(@Nonnull final String videoId) throws ParsingException { - try { - playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); - } catch (final Exception e) { - throw new ParsingException("Could not store JavaScript player", e); - } - } - - private static String getDeobfuscationFuncName(final String thePlayerCode) - throws DeobfuscateException { - Parser.RegexException exception = null; - for (final String regex : REGEXES) { - try { - return Parser.matchGroup1(regex, thePlayerCode); - } catch (final Parser.RegexException re) { - if (exception == null) { - exception = re; - } - } - } - throw new DeobfuscateException( - "Could not find deobfuscate function with any of the given patterns.", exception); - } - - @Nonnull - private static String loadDeobfuscationCode() throws DeobfuscateException { - try { - final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); - - final String functionPattern = "(" - + deobfuscationFunctionName.replace("$", "\\$") - + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; - final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, - playerCode) + ";"; - - final String helperObjectName = - Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", - deobfuscateFunction); - final String helperPattern = - "(var " + helperObjectName.replace("$", "\\$") - + "=\\{.+?\\}\\};)"; - final String helperObject = - Parser.matchGroup1(helperPattern, Objects.requireNonNull(playerCode).replace( - "\n", "")); - - final String callerFunction = - "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " - + deobfuscationFunctionName + "(a);}"; - - return helperObject + deobfuscateFunction + callerFunction; - } catch (final Exception e) { - throw new DeobfuscateException("Could not parse deobfuscate function ", e); - } - } - - @Nonnull - private static String getDeobfuscationCode() throws ParsingException { - if (cachedDeobfuscationCode == null) { - if (isNullOrEmpty(playerCode)) { - throw new ParsingException("playerCode is null"); - } - - cachedDeobfuscationCode = loadDeobfuscationCode(); - } - return cachedDeobfuscationCode; - } - - private static void initStsFromPlayerJsIfNeeded(@Nonnull final String videoId) - throws ParsingException { - if (!isNullOrEmpty(sts)) { - return; - } - if (playerCode == null) { - storePlayerJs(videoId); - if (playerCode == null) { - throw new ParsingException("playerCode is null"); - } - } - sts = Parser.matchGroup1(STS_REGEX, playerCode); - } - - private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException { - final String deobfuscationCode = getDeobfuscationCode(); - - final Context context = Context.enter(); - context.setOptimizationLevel(-1); - final Object result; - try { - final ScriptableObject scope = context.initSafeStandardObjects(); - context.evaluateString(scope, deobfuscationCode, "deobfuscationCode", 1, null); - final Function deobfuscateFunc = (Function) scope.get(DEOBFUSCATION_FUNC_NAME, scope); - result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig}); - } catch (final Exception e) { - throw new DeobfuscateException("Could not get deobfuscate signature", e); - } finally { - Context.exit(); - } - return Objects.toString(result, ""); - } - /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ @@ -1431,14 +1307,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Map- * This is needed for mocks in YouTube stream tests, because when they are ran, the - * {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the - * body present in the mocks is send by the extractor instance. As a result, running all - * YouTube stream tests with the MockDownloader (like the CI does) will fail if this method is - * not called before fetching the page of a test. - *
- */ - public static void resetDeobfuscationCode() { - cachedDeobfuscationCode = null; - playerCode = null; - sts = null; - YoutubeJavaScriptExtractor.resetJavaScriptCode(); - } - /** * Enable or disable the fetch of the Android client for all stream types. * diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java index f7ceef140..9637d300f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java @@ -32,16 +32,16 @@ public class YoutubeJavaScriptExtractorTest { @Test public void testExtractJavaScript__success() throws ParsingException { - String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("d4IGg5dqeO8"); + String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptPlayerCode("d4IGg5dqeO8"); assertPlayerJsCode(playerJsCode); } @Test public void testExtractJavaScript__invalidVideoId__success() throws ParsingException { - String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("not_a_video_id"); + String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptPlayerCode("not_a_video_id"); assertPlayerJsCode(playerJsCode); - playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("11-chars123"); + playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptPlayerCode("11-chars123"); assertPlayerJsCode(playerJsCode); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index fec933c00..ca26eaf35 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.services.DefaultTests; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import javax.annotation.Nullable; import java.util.Collection; @@ -29,7 +28,7 @@ public final class YoutubeTestsUtils { YoutubeParsingHelper.setConsentAccepted(false); YoutubeParsingHelper.resetClientVersionAndKey(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); - YoutubeStreamExtractor.resetDeobfuscationCode(); + YoutubeJavaScriptPlayerManager.clearAllCaches(); } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java deleted file mode 100644 index b87a3213a..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.fail; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mozilla.javascript.EvaluatorException; -import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.io.IOException; - -class YoutubeThrottlingDecrypterTest { - - @BeforeEach - public void setup() throws IOException { - NewPipe.init(DownloaderTestImpl.getInstance()); - } - - @Test - void testExtractFunction__success() throws ParsingException { - final String[] videoIds = {"jE1USQrs1rw", "CqxjzfudGAc", "goH-9MfQI7w", "KYIdr_7H5Yw", "J1WeqmGbYeI"}; - - final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; - - for (final String videoId : videoIds) { - try { - final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, videoId); - assertNotEquals(encryptedUrl, decryptedUrl); - } catch (final EvaluatorException e) { - fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e); - } - } - } - - @Test - void testDecode__success() throws ParsingException { - // URL extracted from browser with the dev tools - final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; - final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, "jE1USQrs1rw"); - // The cipher function changes over time, so we just check if the n param changed. - assertNotEquals(encryptedUrl, decryptedUrl); - } - - @Test - void testDecode__noNParam__success() throws ParsingException { - final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw=="; - final String decrypted = YoutubeThrottlingDecrypter.apply(noNParamUrl, "jE1USQrs1rw"); - - assertEquals(noNParamUrl, decrypted); - } - -} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java new file mode 100644 index 000000000..e6b0d9ad0 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.io.IOException; + +class YoutubeThrottlingParameterDeobfuscationTest { + + @BeforeEach + void setup() throws IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + YoutubeTestsUtils.ensureStateless(); + } + + @Test + void testExtractFunction__success() { + final String[] videoIds = {"jE1USQrs1rw", "CqxjzfudGAc", "goH-9MfQI7w", "KYIdr_7H5Yw", "J1WeqmGbYeI"}; + + final String obfuscatedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; + + for (final String videoId : videoIds) { + try { + final String deobfuscatedUrl = + YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( + videoId, obfuscatedUrl); + assertNotEquals(obfuscatedUrl, deobfuscatedUrl); + } catch (final Exception e) { + fail("Failed to extract throttling parameter deobfuscation function or run its code for video " + + videoId, e); + } + } + } + + @Test + void testDecode__success() throws ParsingException { + // URL extracted from browser with the developer tools + final String obfuscatedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; + final String deobfuscatedUrl = + YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( + "jE1USQrs1rw", obfuscatedUrl); + // The deobfuscation function changes over time, so we just check if the corresponding + // parameter changed + assertNotEquals(obfuscatedUrl, deobfuscatedUrl); + } + + @Test + void testDecode__noThrottlingParam__success() throws ParsingException { + final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw=="; + final String deobfuscatedUrl = + YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( + "jE1USQrs1rw", noNParamUrl); + + assertEquals(noNParamUrl, deobfuscatedUrl); + } +}