be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

-Fixes age-gated video info retrieval. 

-Fixes youtube video stream test passing even when no stream is fetched on age-gated videos.
This commit is contained in:
John Zhen Mo 2017-12-18 14:05:58 -08:00
parent d6a00228e8
commit e0315ca2af
2 changed files with 52 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -543,8 +543,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String GET_VIDEO_INFO_URL = "https://www.youtube.com/get_video_info?video_id=" + "%s" +
"&el=info&ps=default&eurl=&gl=US&hl=en";
private volatile String decryptionCode = "";
@ -559,19 +557,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = getPageHtml(downloader);
final String pageContent = getPageHtml(downloader);
doc = Jsoup.parse(pageContent, getCleanUrl());
String playerUrl;
final String playerUrl;
// TODO: use embedded videos to fetch DASH manifest for all videos
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String infoPageResponse = downloader.download(String.format(GET_VIDEO_INFO_URL, getId()));
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.download(videoInfoUrl);
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = getPlayerUrlFromRestrictedVideo();
playerUrl = info.url;
isAgeRestricted = true;
} else {
JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
final JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
@ -643,24 +643,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private String getPlayerUrlFromRestrictedVideo() throws ParsingException, ReCaptchaException {
@Nonnull
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String embedUrl = "https://www.youtube.com/embed/" + getId();
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
final Downloader downloader = NewPipe.getDownloader();
final String embedUrl = "https://www.youtube.com/embed/" + getId();
final String embedPageContent = downloader.download(embedUrl);
// Get player url
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
// Get embed sts
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
return new EmbeddedInfo(playerUrl, sts);
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
@ -730,10 +732,31 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString();
}
/*//////////////////////////////////////////////////////////////////////////
// Data Class
//////////////////////////////////////////////////////////////////////////*/
private class EmbeddedInfo {
final String url;
final String sts;
EmbeddedInfo(final String url, final String sts) {
this.url = url;
this.sts = sts;
}
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
@Nonnull
private String getVideoInfoUrl(final String id, final String sts) {
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&eurl=https://youtube.googleapis.com/v/" + id +
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();

9
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View File

@ -12,6 +12,8 @@ import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -103,7 +105,12 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
List<VideoStream> streams = new ArrayList<>();
streams.addAll(extractor.getVideoStreams());
streams.addAll(extractor.getVideoOnlyStreams());
assertTrue(streams.size() > 0);
for (VideoStream s : streams) {
assertTrue(s.getUrl(),
s.getUrl().contains(HTTPS));
assertTrue(s.resolution.length() > 0);