be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Cache deobfuscation and improve requests made 

Fix age restriction extraction
Automatically fixes more things
This commit is contained in:
Stypox 2020-10-29 18:44:05 +01:00
parent b42a196f35
commit 827f7bd137
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
3 changed files with 180 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -94,7 +94,7 @@ public class ItagItem {
return item; return item;
} }
} }
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported"); throw new ParsingException("itag=" + itagId + " not supported");
} }
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////

430
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -42,10 +44,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -86,7 +88,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Exceptions // Exceptions
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
public class DeobfuscateException extends ParsingException { public static class DeobfuscateException extends ParsingException {
DeobfuscateException(String message, Throwable cause) { DeobfuscateException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
@ -94,20 +96,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/ /*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
private JsonArray initialAjaxJson; private JsonArray initialAjaxJson;
@Nullable
private JsonObject playerArgs;
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject initialData; private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer; private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer;
private int ageLimit; private int ageLimit = -1;
private boolean newJsonScheme; @Nullable private List<SubtitlesStream> subtitles = null;
@Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
@ -163,8 +162,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
try { // Premiered Feb 21, 2020 try { // Premiered Feb 21, 2020
LocalDate localDate = LocalDate.parse(time, final LocalDate localDate = LocalDate.parse(time,
DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH)); DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) { } catch (Exception ignored) {
} }
@ -224,9 +223,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public int getAgeLimit() { public int getAgeLimit() throws ParsingException {
if (isNullOrEmpty(initialData)) throw new IllegalStateException("initialData is not parsed yet"); if (ageLimit == -1) {
ageLimit = NO_AGE_LIMIT;
final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getArray("rows");
for (final Object metadataRow : metadataRows) {
final JsonArray contents = ((JsonObject) metadataRow)
.getObject("metadataRowRenderer").getArray("contents");
for (final Object content : contents) {
final JsonArray runs = ((JsonObject) content).getArray("runs");
for (final Object run : runs) {
final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
if (rowText.contains("Age-restricted")) {
ageLimit = 18;
return ageLimit;
}
}
}
}
}
return ageLimit; return ageLimit;
} }
@ -313,8 +331,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe); throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
} catch (Exception e) { } catch (Exception e) {
if (ageLimit == 18) return -1; if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get like count", e); throw new ParsingException("Could not get like count", e);
}
return -1;
} }
} }
@ -337,8 +357,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe); throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
} catch (Exception e) { } catch (Exception e) {
if (ageLimit == 18) return -1; if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get dislike count", e); throw new ParsingException("Could not get dislike count", e);
}
return -1;
} }
} }
@ -402,8 +424,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
if (isNullOrEmpty(url)) { if (isNullOrEmpty(url)) {
if (ageLimit == 18) return ""; if (ageLimit == NO_AGE_LIMIT) {
throw new ParsingException("Could not get uploader avatar URL"); throw new ParsingException("Could not get uploader avatar URL");
}
return "";
} }
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
@ -411,19 +435,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public String getSubChannelUrl() throws ParsingException { public String getSubChannelUrl() {
return ""; return "";
} }
@Nonnull @Nonnull
@Override @Override
public String getSubChannelName() throws ParsingException { public String getSubChannelName() {
return ""; return "";
} }
@Nonnull @Nonnull
@Override @Override
public String getSubChannelAvatarUrl() throws ParsingException { public String getSubChannelAvatarUrl() {
return ""; return "";
} }
@ -437,8 +461,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return playerResponse.getObject("streamingData").getString("dashManifestUrl"); return playerResponse.getObject("streamingData").getString("dashManifestUrl");
} else if (videoInfoPage.containsKey("dashmpd")) { } else if (videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd"); dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs != null && playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING);
} else { } else {
return ""; return "";
} }
@ -447,7 +469,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String deobfuscatedSig; String deobfuscatedSig;
deobfuscatedSig = deobfuscateSignature(obfuscatedSig, deobfuscationCode); deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig); dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
} }
@ -465,11 +487,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { try {
return playerResponse.getObject("streamingData").getString("hlsManifestUrl"); return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
} catch (Exception e) { } catch (Exception e) {
if (playerArgs != null && playerArgs.isString("hlsvp")) { throw new ParsingException("Could not get hls manifest url", e);
return playerArgs.getString("hlsvp");
} else {
throw new ParsingException("Could not get hls manifest url", e);
}
} }
} }
@ -535,18 +553,46 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
@Nonnull @Nonnull
public List<SubtitlesStream> getSubtitlesDefault() { public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {
return getSubtitles(MediaFormat.TTML); return getSubtitles(MediaFormat.TTML);
} }
@Override @Override
@Nonnull @Nonnull
public List<SubtitlesStream> getSubtitles(final MediaFormat format) { public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws ParsingException {
assertPageFetched(); assertPageFetched();
List<SubtitlesStream> subtitles = new ArrayList<>(); // If the video is age restricted getPlayerConfig will fail
for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { if (getAgeLimit() != NO_AGE_LIMIT) {
subtitles.add(subtitlesInfo.getSubtitle(format)); return Collections.emptyList();
} }
if (subtitles != null) {
// already calculated
return subtitles;
}
final JsonObject renderer = playerResponse.getObject("captions")
.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// TODO: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
subtitles = new ArrayList<>();
for (int i = 0; i < captionsArray.size(); i++) {
final String languageCode = captionsArray.getObject(i).getString("languageCode");
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
final String vssId = captionsArray.getObject(i).getString("vssId");
if (languageCode != null && baseUrl != null && vssId != null) {
final boolean isAutoGenerated = vssId.startsWith("a.");
final String cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
subtitles.add(new SubtitlesStream(format, languageCode,
cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated));
}
}
return subtitles; return subtitles;
} }
@ -554,14 +600,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public StreamType getStreamType() throws ParsingException { public StreamType getStreamType() throws ParsingException {
assertPageFetched(); assertPageFetched();
try { try {
if (!playerResponse.getObject("streamingData").has(FORMATS) || return playerResponse.getObject("videoDetails").getBoolean("isLiveContent")
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) { ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM;
return StreamType.LIVE_STREAM;
}
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get stream type", e); throw new ParsingException("Could not get stream type", e);
} }
return StreamType.VIDEO_STREAM;
} }
private StreamInfoItemExtractor getNextStream() throws ExtractionException { private StreamInfoItemExtractor getNextStream() throws ExtractionException {
@ -648,48 +691,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("
}; };
private volatile String deobfuscationCode = "";
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader)
final String url = getUrl() + "&pbj=1"; throws IOException, ExtractionException {
final String playerUrl; initialAjaxJson = getJsonResponse(getUrl() + "&pbj=1", getExtractorLocalization());
initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = initialAjaxJson.getObject(3).getObject("response", null);
if (initialData == null) {
if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos initialData = initialAjaxJson.getObject(2).getObject("response", null);
initialData = initialAjaxJson.getObject(2).getObject("response"); if (initialData == null) {
ageLimit = 18; throw new ParsingException("Could not get initial data");
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
} else {
ageLimit = NO_AGE_LIMIT;
JsonObject playerConfig;
initialData = initialAjaxJson.getObject(3).getObject("response");
// sometimes at random YouTube does not provide the player config
playerConfig = initialAjaxJson.getObject(2).getObject("player", null);
if (playerConfig == null) {
newJsonScheme = true;
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
} else {
playerArgs = getPlayerArgs(playerConfig);
playerUrl = getPlayerUrl(playerConfig);
} }
} }
playerResponse = getPlayerResponse(); playerResponse = initialAjaxJson.getObject(2).getObject("playerResponse", null);
if (playerResponse == null || !playerResponse.has("streamingData")) {
// try to get player response by fetching video info page
fetchVideoInfoPage();
}
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
final String status = playabilityStatus.getString("status"); final String status = playabilityStatus.getString("status");
@ -698,117 +717,77 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String reason = playabilityStatus.getString("reason"); final String reason = playabilityStatus.getString("reason");
throw new ContentNotAvailableException("Got error: \"" + reason + "\""); throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
} }
if (deobfuscationCode.isEmpty()) {
deobfuscationCode = loadDeobfuscationCode(playerUrl);
}
if (subtitlesInfos.isEmpty()) {
subtitlesInfos.addAll(getAvailableSubtitlesInfo());
}
} }
private JsonObject getPlayerArgs(final JsonObject playerConfig) throws ParsingException { private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
//attempt to load the youtube js player JSON arguments final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
final JsonObject playerArgs = playerConfig.getObject("args", null); final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
if (playerArgs == null) { final String infoPageResponse = NewPipe.getDownloader()
throw new ParsingException("Could not extract args from YouTube player config"); .get(videoInfoUrl, getExtractorLocalization()).responseBody();
} videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
return playerArgs;
}
private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for deobfuscating cryptic signatures inside certain stream URLs.
final String playerUrl = playerConfig.getObject("assets").getString("js");
if (playerUrl == null) {
throw new ParsingException("Could not extract js URL from YouTube player config");
} else if (playerUrl.startsWith("//")) {
return HTTPS + playerUrl;
}
return playerUrl;
}
private JsonObject getPlayerResponse() throws ParsingException {
try { try {
String playerResponseStr; playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
if (newJsonScheme) { } catch (JsonParserException e) {
return initialAjaxJson.getObject(2).getObject("playerResponse"); throw new ParsingException(
} "Could not parse YouTube player response from video info page", e);
if (playerArgs != null) {
playerResponseStr = playerArgs.getString("player_response");
} else {
playerResponseStr = videoInfoPage.get("player_response");
}
return JsonParser.object().from(playerResponseStr);
} catch (Exception e) {
throw new ParsingException("Could not parse yt player response", e);
} }
} }
@Nonnull @Nonnull
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
try { try {
final Downloader downloader = NewPipe.getDownloader();
final String embedUrl = "https://www.youtube.com/embed/" + getId(); final String embedUrl = "https://www.youtube.com/embed/" + getId();
final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody(); final String embedPageContent = NewPipe.getDownloader()
.get(embedUrl, getExtractorLocalization()).responseBody();
// Get player url
String playerUrl = null;
try { try {
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent) playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", ""); .replace("\\", "").replace("\"", "");
} catch (Parser.RegexException ex) { } catch (Parser.RegexException ex) {
// playerUrl is still available in the file, just somewhere else // playerJsUrl is still available in the file, just somewhere else TODO
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
final Document doc = Jsoup.parse(embedPageContent); final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base"); final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (Element elem : elems) { for (Element elem : elems) {
if (elem.attr("src").contains("base.js")) { if (elem.attr("src").contains("base.js")) {
playerUrl = elem.attr("src"); playerJsUrl = elem.attr("src");
break;
} }
} }
if (playerUrl == null) {
throw new ParsingException("Could not get playerUrl");
}
} }
if (playerUrl.startsWith("//")) { // Get embed sts
playerUrl = HTTPS + playerUrl; return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent);
} else if (playerUrl.startsWith("/")) { } catch (Exception i) {
playerUrl = HTTPS + "//youtube.com" + playerUrl; // if it fails we simply reply with no sts as then it does not seem to be necessary
} return "";
try {
// Get embed sts
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
return new EmbeddedInfo(playerUrl, sts);
} catch (Exception i) {
// if it fails we simply reply with no sts as then it does not seem to be necessary
return new EmbeddedInfo(playerUrl, "");
}
} catch (IOException e) {
throw new ParsingException(
"Could not load deobfuscation code from YouTube video embed", e);
} }
} }
private String loadDeobfuscationCode(String playerUrl) throws DeobfuscateException {
try {
Downloader downloader = NewPipe.getDownloader();
if (!playerUrl.contains("https://youtube.com")) {
//sometimes the https://youtube.com part does not get send with
//than we have to add it by hand
playerUrl = "https://youtube.com" + playerUrl;
}
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
return Parser.matchGroup1(regex, playerCode);
} catch (Parser.RegexException re) {
if (exception == null) {
exception = re;
}
}
}
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
}
private String loadDeobfuscationCode(@Nonnull final String playerJsUrl)
throws DeobfuscateException {
try {
final String playerCode = NewPipe.getDownloader()
.get(playerJsUrl, getExtractorLocalization()).responseBody();
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
final String functionPattern = "(" final String functionPattern = "("
@ -834,7 +813,34 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
private String deobfuscateSignature(String obfuscatedSig, String deobfuscationCode) throws DeobfuscateException { @Nonnull
private String getDeobfuscationCode() throws ParsingException {
if (cachedDeobfuscationCode == null) {
if (playerJsUrl == null) {
// the currentPlayerJsUrl was not found in any page fetched so far and there is
// nothing cached, so try fetching embedded info
getEmbeddedInfoStsAndStorePlayerJsUrl();
if (playerJsUrl == null) {
throw new ParsingException(
"Embedded info did not provide YouTube player js url");
}
}
if (playerJsUrl.startsWith("//")) {
playerJsUrl = HTTPS + playerJsUrl;
} else if (playerJsUrl.startsWith("/")) {
// sometimes https://youtube.com part has to be added manually
playerJsUrl = HTTPS + "//youtube.com" + playerJsUrl;
}
cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl);
}
return cachedDeobfuscationCode;
}
private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException {
final String deobfuscationCode = getDeobfuscationCode();
final Context context = Context.enter(); final Context context = Context.enter();
context.setOptimizationLevel(-1); context.setOptimizationLevel(-1);
final Object result; final Object result;
@ -851,88 +857,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString(); return result == null ? "" : result.toString();
} }
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
return Parser.matchGroup1(regex, playerCode);
} catch (Parser.RegexException re) {
if (exception == null) {
exception = re;
}
}
}
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
}
@Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
// If the video is age restricted getPlayerConfig will fail
if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
final JsonObject captions;
if (!playerResponse.has("captions")) {
// Captions does not exist
return Collections.emptyList();
}
captions = playerResponse.getObject("captions");
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// todo: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
// This check is necessary since there may be cases where subtitles metadata do not contain caption track info
// e.g. https://www.youtube.com/watch?v=-Vpwatutnko
final int captionsSize = captionsArray.size();
if (captionsSize == 0) return Collections.emptyList();
List<SubtitlesInfo> result = new ArrayList<>();
for (int i = 0; i < captionsSize; i++) {
final String languageCode = captionsArray.getObject(i).getString("languageCode");
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
final String vssId = captionsArray.getObject(i).getString("vssId");
if (languageCode != null && baseUrl != null && vssId != null) {
final boolean isAutoGenerated = vssId.startsWith("a.");
result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
}
}
return result;
}
/*//////////////////////////////////////////////////////////////////////////
// Data Class
//////////////////////////////////////////////////////////////////////////*/
private class EmbeddedInfo {
final String url;
final String sts;
EmbeddedInfo(final String url, final String sts) {
this.url = url;
this.sts = sts;
}
}
private class SubtitlesInfo {
final String cleanUrl;
final String languageCode;
final boolean isGenerated;
public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
this.cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
this.languageCode = languageCode;
this.isGenerated = isGenerated;
}
public SubtitlesStream getSubtitle(final MediaFormat format) {
return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated);
}
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Utils // Utils
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -989,14 +913,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"&sts=" + sts + "&ps=default&gl=US&hl=en"; "&sts=" + sts + "&ps=default&gl=US&hl=en";
} }
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { private Map<String, ItagItem> getItags(final String streamingDataKey,
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>(); final ItagItem.ItagType itagTypeWanted)
JsonObject streamingData = playerResponse.getObject("streamingData"); throws ParsingException {
final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
final JsonObject streamingData = playerResponse.getObject("streamingData");
if (!streamingData.has(streamingDataKey)) { if (!streamingData.has(streamingDataKey)) {
return urlAndItags; return urlAndItags;
} }
JsonArray formats = streamingData.getArray(streamingDataKey); final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) { for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i); JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag"); int itag = formatData.getInt("itag");
@ -1022,7 +948,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
: formatData.getString("signatureCipher"); : formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString); final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"), deobfuscationCode); + deobfuscateSignature(cipher.get("s"));
} }
urlAndItags.put(streamUrl, itagItem); urlAndItags.put(streamUrl, itagItem);

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java
View File

@ -209,6 +209,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
final StreamInfoItemsCollector relatedStreams = extractor().getRelatedStreams(); final StreamInfoItemsCollector relatedStreams = extractor().getRelatedStreams();
if (expectedHasRelatedStreams()) { if (expectedHasRelatedStreams()) {
assertNotNull(relatedStreams);
defaultTestListOfItems(extractor().getService(), relatedStreams.getItems(), defaultTestListOfItems(extractor().getService(), relatedStreams.getItems(),
relatedStreams.getErrors()); relatedStreams.getErrors());
} else { } else {