Make YoutubeJavaScriptExtractor and JavaScript methods static
Also address review and rewrite some comments
This commit is contained in:
parent
a683c8d278
commit
3a3d1d7f2b
|
@ -12,47 +12,51 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
import javax.annotation.Nonnull;
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Youtube restricts streaming their media in multiple ways by requiring clients to apply a cipher function
|
* YouTube restricts streaming their media in multiple ways by requiring clients to apply a cipher function
|
||||||
* on parameters of requests.
|
* on parameters of requests.
|
||||||
* The cipher function is sent alongside as a JavaScript function.
|
* The cipher function is sent alongside as a JavaScript function.
|
||||||
* <p>
|
* <p>
|
||||||
* This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions.
|
* This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions.
|
||||||
*/
|
*/
|
||||||
public class YoutubeJavascriptExtractor {
|
public class YoutubeJavaScriptExtractor {
|
||||||
|
|
||||||
private static final String HTTPS = "https:";
|
private static final String HTTPS = "https:";
|
||||||
private static String cachedJavascriptCode;
|
private static String cachedJavaScriptCode;
|
||||||
|
|
||||||
|
private YoutubeJavaScriptExtractor() {
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls.
|
* Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls.
|
||||||
*
|
*
|
||||||
* @param videoId Does not influence the result, but a valid video id can prevent tracking
|
* @param videoId Does not influence the result, but a valid video id may help in the chance that YouTube tracks it.
|
||||||
* @return The whole javascript file as a string.
|
* @return The whole javascript file as a string.
|
||||||
* @throws ParsingException If the extraction failed.
|
* @throws ParsingException If the extraction failed.
|
||||||
*/
|
*/
|
||||||
@Nonnull
|
@Nonnull
|
||||||
public static String extractJavascriptCode(String videoId) throws ParsingException {
|
public static String extractJavaScriptCode(String videoId) throws ParsingException {
|
||||||
if (cachedJavascriptCode == null) {
|
if (cachedJavaScriptCode == null) {
|
||||||
final YoutubeJavascriptExtractor extractor = new YoutubeJavascriptExtractor();
|
final String playerJsUrl = YoutubeJavaScriptExtractor.cleanJavaScriptUrl(
|
||||||
String playerJsUrl = extractor.cleanJavascriptUrl(extractor.extractJavascriptUrl(videoId));
|
YoutubeJavaScriptExtractor.extractJavaScriptUrl(videoId));
|
||||||
cachedJavascriptCode = extractor.downloadJavascriptCode(playerJsUrl);
|
cachedJavaScriptCode = YoutubeJavaScriptExtractor.downloadJavaScriptCode(playerJsUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
return cachedJavascriptCode;
|
return cachedJavaScriptCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Same as {@link YoutubeJavascriptExtractor#extractJavascriptCode(String)} but with a constant value for videoId.
|
* Same as {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)} but with a constant value for videoId.
|
||||||
* Possible because the videoId has no influence on the result.
|
* Possible because the videoId has no influence on the result.
|
||||||
*
|
* <p>
|
||||||
* For tracking avoidance purposes it may make sense to pass in valid video ids.
|
* In the off chance that YouTube tracks with which video id the request is made, it may make sense to pass in
|
||||||
|
* video ids.
|
||||||
*/
|
*/
|
||||||
@Nonnull
|
@Nonnull
|
||||||
public static String extractJavascriptCode() throws ParsingException {
|
public static String extractJavaScriptCode() throws ParsingException {
|
||||||
return extractJavascriptCode("d4IGg5dqeO8");
|
return extractJavaScriptCode("d4IGg5dqeO8");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractJavascriptUrl(String videoId) throws ParsingException {
|
private static String extractJavaScriptUrl(String videoId) throws ParsingException {
|
||||||
try {
|
try {
|
||||||
final String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
final String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||||
final String embedPageContent = NewPipe.getDownloader()
|
final String embedPageContent = NewPipe.getDownloader()
|
||||||
|
@ -80,7 +84,7 @@ public class YoutubeJavascriptExtractor {
|
||||||
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String cleanJavascriptUrl(String playerJsUrl) {
|
private static String cleanJavaScriptUrl(String playerJsUrl) {
|
||||||
if (playerJsUrl.startsWith("//")) {
|
if (playerJsUrl.startsWith("//")) {
|
||||||
return HTTPS + playerJsUrl;
|
return HTTPS + playerJsUrl;
|
||||||
} else if (playerJsUrl.startsWith("/")) {
|
} else if (playerJsUrl.startsWith("/")) {
|
||||||
|
@ -91,7 +95,7 @@ public class YoutubeJavascriptExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String downloadJavascriptCode(String playerJsUrl) throws ParsingException {
|
private static String downloadJavaScriptCode(String playerJsUrl) throws ParsingException {
|
||||||
try {
|
try {
|
||||||
return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody();
|
return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
|
@ -1,7 +1,7 @@
|
||||||
package org.schabi.newpipe.extractor.services.youtube;
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.utils.Javascript;
|
import org.schabi.newpipe.extractor.utils.JavaScript;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
@ -35,14 +35,14 @@ public class YoutubeThrottlingDecrypter {
|
||||||
* Otherwise use the no-arg constructor which uses a constant value.
|
* Otherwise use the no-arg constructor which uses a constant value.
|
||||||
*/
|
*/
|
||||||
public YoutubeThrottlingDecrypter(String videoId) throws ParsingException {
|
public YoutubeThrottlingDecrypter(String videoId) throws ParsingException {
|
||||||
final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(videoId);
|
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
|
||||||
|
|
||||||
functionName = parseDecodeFunctionName(playerJsCode);
|
functionName = parseDecodeFunctionName(playerJsCode);
|
||||||
function = parseDecodeFunction(playerJsCode, functionName);
|
function = parseDecodeFunction(playerJsCode, functionName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public YoutubeThrottlingDecrypter() throws ParsingException {
|
public YoutubeThrottlingDecrypter() throws ParsingException {
|
||||||
final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode();
|
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
|
||||||
|
|
||||||
functionName = parseDecodeFunctionName(playerJsCode);
|
functionName = parseDecodeFunctionName(playerJsCode);
|
||||||
function = parseDecodeFunction(playerJsCode, functionName);
|
function = parseDecodeFunction(playerJsCode, functionName);
|
||||||
|
@ -78,8 +78,7 @@ public class YoutubeThrottlingDecrypter {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String decryptNParam(String nParam) {
|
private String decryptNParam(String nParam) {
|
||||||
Javascript javascript = new Javascript();
|
return JavaScript.run(function, functionName, nParam);
|
||||||
return javascript.run(function, functionName, nParam);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String replaceNParam(String url, String oldValue, String newValue) {
|
private String replaceNParam(String url, String oldValue, String newValue) {
|
||||||
|
|
|
@ -4,10 +4,6 @@ import com.grack.nanojson.JsonArray;
|
||||||
import com.grack.nanojson.JsonObject;
|
import com.grack.nanojson.JsonObject;
|
||||||
import com.grack.nanojson.JsonParser;
|
import com.grack.nanojson.JsonParser;
|
||||||
import com.grack.nanojson.JsonParserException;
|
import com.grack.nanojson.JsonParserException;
|
||||||
import org.jsoup.Jsoup;
|
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
import org.jsoup.select.Elements;
|
|
||||||
import org.mozilla.javascript.Context;
|
import org.mozilla.javascript.Context;
|
||||||
import org.mozilla.javascript.Function;
|
import org.mozilla.javascript.Function;
|
||||||
import org.mozilla.javascript.ScriptableObject;
|
import org.mozilla.javascript.ScriptableObject;
|
||||||
|
@ -24,7 +20,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
|
||||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||||
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
|
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavascriptExtractor;
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||||
|
@ -524,7 +520,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||||
assertPageFetched();
|
assertPageFetched();
|
||||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||||
YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||||
|
@ -817,7 +813,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
private String loadDeobfuscationCode()
|
private String loadDeobfuscationCode()
|
||||||
throws DeobfuscateException {
|
throws DeobfuscateException {
|
||||||
try {
|
try {
|
||||||
final String playerCode = YoutubeJavascriptExtractor.extractJavascriptCode(getId());
|
final String playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(getId());
|
||||||
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
|
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
|
||||||
|
|
||||||
final String functionPattern = "("
|
final String functionPattern = "("
|
||||||
|
|
|
@ -4,9 +4,12 @@ import org.mozilla.javascript.Context;
|
||||||
import org.mozilla.javascript.Function;
|
import org.mozilla.javascript.Function;
|
||||||
import org.mozilla.javascript.ScriptableObject;
|
import org.mozilla.javascript.ScriptableObject;
|
||||||
|
|
||||||
public class Javascript {
|
public class JavaScript {
|
||||||
|
|
||||||
public String run(String function, String functionName, String... parameters) {
|
private JavaScript() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String run(String function, String functionName, String... parameters) {
|
||||||
try {
|
try {
|
||||||
Context context = Context.enter();
|
Context context = Context.enter();
|
||||||
context.setOptimizationLevel(-1);
|
context.setOptimizationLevel(-1);
|
|
@ -12,7 +12,7 @@ import static org.hamcrest.CoreMatchers.allOf;
|
||||||
import static org.hamcrest.CoreMatchers.containsString;
|
import static org.hamcrest.CoreMatchers.containsString;
|
||||||
import static org.hamcrest.MatcherAssert.assertThat;
|
import static org.hamcrest.MatcherAssert.assertThat;
|
||||||
|
|
||||||
public class YoutubeJavascriptExtractorTest {
|
public class YoutubeJavaScriptExtractorTest {
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setup() throws IOException {
|
public void setup() throws IOException {
|
||||||
|
@ -20,20 +20,20 @@ public class YoutubeJavascriptExtractorTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExtractJavascript__success() throws ParsingException {
|
public void testExtractJavaScript__success() throws ParsingException {
|
||||||
String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("d4IGg5dqeO8");
|
String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("d4IGg5dqeO8");
|
||||||
assertPlayerJsCode(playerJsCode);
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode();
|
playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
|
||||||
assertPlayerJsCode(playerJsCode);
|
assertPlayerJsCode(playerJsCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExtractJavascript__invalidVideoId__success() throws ParsingException {
|
public void testExtractJavaScript__invalidVideoId__success() throws ParsingException {
|
||||||
String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("not_a_video_id");
|
String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("not_a_video_id");
|
||||||
assertPlayerJsCode(playerJsCode);
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("11-chars123");
|
playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode("11-chars123");
|
||||||
assertPlayerJsCode(playerJsCode);
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
}
|
}
|
Loading…
Reference in New Issue