From 29a4cc78bf71b1f443b7bbc220f3769f03c914e5 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Thu, 23 Jan 2020 21:20:38 +0100 Subject: [PATCH 1/4] Fix typo in decryption constant --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ede..8466d5d1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -677,7 +677,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; - private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX = + private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; @@ -873,7 +873,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // to se if the function can actually be found with the given regex. // However if this cascading should propably be cleaned up somehow as it looks a bit weird. try { - decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode); + decryptionFunctionName = Parser.matchGroup1(DECRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode); } catch (Parser.RegexException re) { try { decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode); From 9b45c61103e169dad440d2207222fd3c929cad41 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 24 Jan 2020 01:27:54 +0100 Subject: [PATCH 2/4] Typos, comments and formatting --- .../java/org/schabi/newpipe/extractor/Extractor.java | 2 +- .../youtube/extractors/YoutubeStreamExtractor.java | 10 +++++----- .../org/schabi/newpipe/extractor/utils/Parser.java | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index e5c03d3f1..b6e90d53c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -56,7 +56,7 @@ public abstract class Extractor{ } protected void assertPageFetched() { - if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()"); + if (!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()"); } protected boolean isPageFetched() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8466d5d1a..db3bc4dcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -773,7 +773,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject getPlayerResponse() throws ParsingException { try { String playerResponseStr; - if(playerArgs != null) { + if (playerArgs != null) { playerResponseStr = playerArgs.getString("player_response"); } else { playerResponseStr = videoInfoPage.get("player_response"); @@ -805,7 +805,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String sts = Parser.matchGroup1(stsPattern, embedPageContent); return new EmbeddedInfo(playerUrl, sts); } catch (Exception i) { - // if it failes we simply reply with no sts as then it does not seem to be necessary + // if it fails we simply reply with no sts as then it does not seem to be necessary return new EmbeddedInfo(playerUrl, ""); } @@ -871,7 +871,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String decryptionFunctionName; // Cascading things in catch is ugly, but its faster than running a match before getting the actual name // to se if the function can actually be found with the given regex. - // However if this cascading should propably be cleaned up somehow as it looks a bit weird. + // However if this cascading should probably be cleaned up somehow as it looks a bit weird. try { decryptionFunctionName = Parser.matchGroup1(DECRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode); } catch (Parser.RegexException re) { @@ -891,7 +891,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private List getAvailableSubtitlesInfo() throws SubtitlesException { // If the video is age restricted getPlayerConfig will fail - if(isAgeRestricted) return Collections.emptyList(); + if (isAgeRestricted) return Collections.emptyList(); final JsonObject captions; if (!playerResponse.has("captions")) { @@ -908,7 +908,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // This check is necessary since there may be cases where subtitles metadata do not contain caption track info // e.g. https://www.youtube.com/watch?v=-Vpwatutnko final int captionsSize = captionsArray.size(); - if(captionsSize == 0) return Collections.emptyList(); + if (captionsSize == 0) return Collections.emptyList(); List result = new ArrayList<>(); for (int i = 0; i < captionsSize; i++) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 6cd938975..b03e304c5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -67,6 +67,7 @@ public class Parser { if (foundMatch) { return mat.group(group); } else { + // only pass input to exception message when it is not too long if (input.length() > 1024) { throw new RegexException("failed to find pattern \"" + pat.pattern()); } else { From bce87f3d01a3717c330eda880a6e60990e698727 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 24 Jan 2020 01:30:54 +0100 Subject: [PATCH 3/4] Improve getDescriptionFuncName by removing catches and adding a loop --- .../extractors/YoutubeStreamExtractor.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index db3bc4dcd..98df78f74 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -868,24 +868,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private String getDecryptionFuncName(String playerCode) throws DecryptException { - String decryptionFunctionName; - // Cascading things in catch is ugly, but its faster than running a match before getting the actual name - // to se if the function can actually be found with the given regex. - // However if this cascading should probably be cleaned up somehow as it looks a bit weird. - try { - decryptionFunctionName = Parser.matchGroup1(DECRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode); - } catch (Parser.RegexException re) { + String[] decryptionFuncNameRegexes = { + DECRYPTION_SIGNATURE_FUNCTION_REGEX, + DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, + DECRYPTION_AKAMAIZED_STRING_REGEX + }; + Parser.RegexException exception = null; + for (String regex : decryptionFuncNameRegexes) { try { - decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode); - } catch (Parser.RegexException re2) { - try { - decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode); - } catch (Parser.RegexException re3) { - throw new DecryptException("Could not find decrypt function with any of the given patterns.", re); - } + return Parser.matchGroup1(regex, playerCode); + } catch (Parser.RegexException re) { + if (exception == null) + exception = re; } } - return decryptionFunctionName; + throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception); } @Nonnull From 7b72fd2a7d3632f3de0c57d525664c4c26c9af19 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 24 Jan 2020 01:59:39 +0100 Subject: [PATCH 4/4] [YouTube] Add new decryption function --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 98df78f74..cb2a5f53a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -679,6 +679,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; + private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = + "\\b([\\w$]{2})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX = @@ -697,7 +699,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; // Check if the video is age restricted - if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { + Elements e = doc.select("meta[property=\"og:restrictions:age\"]"); + if (!e.isEmpty()) { final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); @@ -869,6 +872,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String getDecryptionFuncName(String playerCode) throws DecryptException { String[] decryptionFuncNameRegexes = { + DECRYPTION_SIGNATURE_FUNCTION_REGEX_2, DECRYPTION_SIGNATURE_FUNCTION_REGEX, DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, DECRYPTION_AKAMAIZED_STRING_REGEX