[Youtube] Make throttling decryption more resilient to api change
This commit is contained in:
XiangRongLin
parent
65129e65a8
commit
545522b80f
|
|
@ -38,7 +38,9 @@ public class YoutubeThrottlingDecrypter {
|
|||
private static final Pattern FUNCTION_NAME_PATTERN = Pattern.compile(
|
||||
"b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\S+)\\(b\\),a\\.set\\(\"n\",b\\)");
|
||||
|
||||
private static final Map<String, String> nParams = new HashMap<>();
|
||||
private static final Map<String, String> N_PARAMS_CACHE = new HashMap<>();
|
||||
private static String FUNCTION;
|
||||
private static String FUNCTION_NAME;
|
||||
|
||||
private final String functionName;
|
||||
private final String function;
|
||||
|
|
@ -49,6 +51,8 @@ public class YoutubeThrottlingDecrypter {
|
|||
* is requested.
|
||||
* </p>
|
||||
* Otherwise use the no-arg constructor which uses a constant value.
|
||||
*
|
||||
* @deprecated Use static function instead
|
||||
*/
|
||||
public YoutubeThrottlingDecrypter(final String videoId) throws ParsingException {
|
||||
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
|
||||
|
|
@ -57,6 +61,9 @@ public class YoutubeThrottlingDecrypter {
|
|||
function = parseDecodeFunction(playerJsCode, functionName);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use static function instead
|
||||
*/
|
||||
public YoutubeThrottlingDecrypter() throws ParsingException {
|
||||
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
|
||||
|
||||
|
|
@ -64,27 +71,50 @@ public class YoutubeThrottlingDecrypter {
|
|||
function = parseDecodeFunction(playerJsCode, functionName);
|
||||
}
|
||||
|
||||
private String parseDecodeFunctionName(final String playerJsCode)
|
||||
throws Parser.RegexException {
|
||||
String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode);
|
||||
final int arrayStartBrace = functionName.indexOf("[");
|
||||
|
||||
if (arrayStartBrace > 0) {
|
||||
final String arrayVarName = functionName.substring(0, arrayStartBrace);
|
||||
final String order = functionName.substring(
|
||||
arrayStartBrace + 1, functionName.indexOf("]"));
|
||||
final int arrayNum = Integer.parseInt(order);
|
||||
final Pattern arrayPattern = Pattern.compile(
|
||||
String.format("var %s=\\[(.+?)\\];", arrayVarName));
|
||||
final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode);
|
||||
final String[] names = arrayStr.split(",");
|
||||
functionName = names[arrayNum];
|
||||
/**
|
||||
* <p>
|
||||
* The videoId is only used to fetch the decryption function.
|
||||
* It can be a constant value of any existing video.
|
||||
* A constant value is discouraged, because it could allow tracking.
|
||||
*/
|
||||
public static String apply(final String url, final String videoId) throws ParsingException {
|
||||
if (containsNParam(url)) {
|
||||
if (FUNCTION == null) {
|
||||
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
|
||||
|
||||
FUNCTION_NAME = parseDecodeFunctionName(playerJsCode);
|
||||
FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
|
||||
}
|
||||
return functionName;
|
||||
|
||||
final String oldNParam = parseNParam(url);
|
||||
final String newNParam = decryptNParam(FUNCTION, FUNCTION_NAME, oldNParam);
|
||||
return replaceNParam(url, oldNParam, newNParam);
|
||||
} else {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
private static String parseDecodeFunctionName(final String playerJsCode)
|
||||
throws Parser.RegexException {
|
||||
String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode);
|
||||
final int arrayStartBrace = functionName.indexOf("[");
|
||||
|
||||
if (arrayStartBrace > 0) {
|
||||
final String arrayVarName = functionName.substring(0, arrayStartBrace);
|
||||
final String order = functionName.substring(
|
||||
arrayStartBrace + 1, functionName.indexOf("]"));
|
||||
final int arrayNum = Integer.parseInt(order);
|
||||
final Pattern arrayPattern = Pattern.compile(
|
||||
String.format("var %s=\\[(.+?)\\];", arrayVarName));
|
||||
final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode);
|
||||
final String[] names = arrayStr.split(",");
|
||||
functionName = names[arrayNum];
|
||||
}
|
||||
return functionName;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String parseDecodeFunction(final String playerJsCode, final String functionName)
|
||||
private static String parseDecodeFunction(final String playerJsCode, final String functionName)
|
||||
throws Parser.RegexException {
|
||||
try {
|
||||
return parseWithParenthesisMatching(playerJsCode, functionName);
|
||||
|
|
@ -94,49 +124,50 @@ public class YoutubeThrottlingDecrypter {
|
|||
}
|
||||
|
||||
@Nonnull
|
||||
private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
|
||||
private static String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
|
||||
final String functionBase = functionName + "=function";
|
||||
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
|
||||
private static String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
|
||||
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
|
||||
Pattern.DOTALL);
|
||||
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
|
||||
}
|
||||
|
||||
public String apply(final String url) throws Parser.RegexException {
|
||||
@Deprecated
|
||||
public String apply(final String url) throws ParsingException {
|
||||
if (containsNParam(url)) {
|
||||
final String oldNParam = parseNParam(url);
|
||||
final String newNParam = decryptNParam(oldNParam);
|
||||
final String newNParam = decryptNParam(function, functionName, oldNParam);
|
||||
return replaceNParam(url, oldNParam, newNParam);
|
||||
} else {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean containsNParam(final String url) {
|
||||
private static boolean containsNParam(final String url) {
|
||||
return Parser.isMatch(N_PARAM_PATTERN, url);
|
||||
}
|
||||
|
||||
private String parseNParam(final String url) throws Parser.RegexException {
|
||||
private static String parseNParam(final String url) throws Parser.RegexException {
|
||||
return Parser.matchGroup1(N_PARAM_PATTERN, url);
|
||||
}
|
||||
|
||||
private String decryptNParam(final String nParam) {
|
||||
if (nParams.containsKey(nParam)) {
|
||||
return nParams.get(nParam);
|
||||
private static String decryptNParam(final String function, final String functionName, final String nParam) {
|
||||
if (N_PARAMS_CACHE.containsKey(nParam)) {
|
||||
return N_PARAMS_CACHE.get(nParam);
|
||||
}
|
||||
final String decryptedNParam = JavaScript.run(function, functionName, nParam);
|
||||
nParams.put(nParam, decryptedNParam);
|
||||
N_PARAMS_CACHE.put(nParam, decryptedNParam);
|
||||
return decryptedNParam;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String replaceNParam(@Nonnull final String url,
|
||||
final String oldValue,
|
||||
final String newValue) {
|
||||
private static String replaceNParam(@Nonnull final String url,
|
||||
final String oldValue,
|
||||
final String newValue) {
|
||||
return url.replace(oldValue, newValue);
|
||||
}
|
||||
|
||||
|
|
@ -144,13 +175,13 @@ public class YoutubeThrottlingDecrypter {
|
|||
* @return the number of the cached "n" query parameters.
|
||||
*/
|
||||
public static int getCacheSize() {
|
||||
return nParams.size();
|
||||
return N_PARAMS_CACHE.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all stored "n" query parameters.
|
||||
*/
|
||||
public static void clearCache() {
|
||||
nParams.clear();
|
||||
N_PARAMS_CACHE.clear();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -469,14 +469,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
|
||||
ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
String url = entry.getKey();
|
||||
url = throttlingDecrypter.apply(url);
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final AudioStream audioStream = new AudioStream(url, itag);
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
|
|
@ -494,14 +492,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
|
||||
ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
String url = entry.getKey();
|
||||
url = throttlingDecrypter.apply(url);
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final VideoStream videoStream = new VideoStream(url, false, itag);
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
|
|
@ -519,14 +515,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
|
||||
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
String url = entry.getKey();
|
||||
url = throttlingDecrypter.apply(url);
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final VideoStream videoStream = new VideoStream(url, true, itag);
|
||||
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
|
||||
|
|
@ -540,6 +534,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return videoOnlyStreams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to decrypt url and fallback to given url, because decryption is not
|
||||
* always needed.
|
||||
* This way a breaking change from YouTube does not result in a broken extractor.
|
||||
*/
|
||||
private String tryDecryption(final String url, final String videoId) {
|
||||
try {
|
||||
return YoutubeThrottlingDecrypter.apply(url, videoId);
|
||||
} catch (ParsingException e) {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Nonnull
|
||||
public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ public class YoutubeThrottlingDecrypterTest {
|
|||
|
||||
for (final String videoId : videoIds) {
|
||||
try {
|
||||
final String decryptedUrl = new YoutubeThrottlingDecrypter(videoId).apply(encryptedUrl);
|
||||
final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, videoId);
|
||||
assertNotEquals(encryptedUrl, decryptedUrl);
|
||||
} catch (EvaluatorException e) {
|
||||
fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e);
|
||||
|
|
@ -40,7 +40,7 @@ public class YoutubeThrottlingDecrypterTest {
|
|||
public void testDecode__success() throws ParsingException {
|
||||
// URL extracted from browser with the dev tools
|
||||
final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190";
|
||||
final String decryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl);
|
||||
final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, "jE1USQrs1rw");
|
||||
// The cipher function changes over time, so we just check if the n param changed.
|
||||
assertNotEquals(encryptedUrl, decryptedUrl);
|
||||
}
|
||||
|
|
@ -48,7 +48,7 @@ public class YoutubeThrottlingDecrypterTest {
|
|||
@Test
|
||||
public void testDecode__noNParam__success() throws ParsingException {
|
||||
final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw==";
|
||||
final String decrypted = new YoutubeThrottlingDecrypter().apply(noNParamUrl);
|
||||
final String decrypted = YoutubeThrottlingDecrypter.apply(noNParamUrl, "jE1USQrs1rw");
|
||||
|
||||
assertEquals(noNParamUrl, decrypted);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue