From e851c12230377d9d661cfeb40fe6c4ddef259eb9 Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Sun, 25 Feb 2018 12:34:09 -0800 Subject: [PATCH 1/6] -Removed live stream exceptions --- .../extractor/services/youtube/YoutubeStreamExtractor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 346ea1184..cb8e1ca98 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -594,9 +594,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception e) { throw new ParsingException("Could not parse yt player config", e); } - if (isLiveStream) { - throw new LiveStreamException("This is a Live stream. Can't use those right now."); - } return playerArgs; } From 690f241357242da2465ed2904f14d91a01d76c7d Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Sun, 25 Feb 2018 13:03:32 -0800 Subject: [PATCH 2/6] -Added hls manifest url extraction. -Changed manifest url fields to nonnull. --- .../soundcloud/SoundcloudStreamExtractor.java | 9 ++++++- .../youtube/YoutubeStreamExtractor.java | 25 ++++++++++++++----- .../extractor/stream/StreamExtractor.java | 3 ++- .../newpipe/extractor/stream/StreamInfo.java | 15 +++++++++++ 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 28353c97a..6a14f3b13 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -116,9 +116,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return SoundcloudParsingHelper.getAvatarUrl(track); } + @Nonnull @Override public String getDashMpdUrl() { - return null; + return ""; + } + + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + return ""; } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index cb8e1ca98..38b94bdc6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -66,12 +66,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - public class LiveStreamException extends ContentNotAvailableException { - LiveStreamException(String message) { - super(message); - } - } - public class SubtitlesException extends ContentNotAvailableException { SubtitlesException(String message, Throwable cause) { super(message, cause); @@ -338,6 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull @Override public String getDashMpdUrl() throws ParsingException { assertPageFetched(); @@ -365,6 +360,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + assertPageFetched(); + try { + String hlsvp; + if (playerArgs != null && playerArgs.isString("hlsvp")) { + hlsvp = playerArgs.getString("hlsvp", ""); + } else { + return ""; + } + + return hlsvp; + } catch (Exception e) { + throw new ParsingException("Could not get hls manifest url", e); + } + } + @Override public List getAudioStreams() throws IOException, ExtractionException { assertPageFetched(); diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 05317fb65..b54874afb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -126,7 +126,8 @@ public abstract class StreamExtractor extends Extractor { * @return the url as a string or an empty string * @throws ParsingException if an error occurs while reading */ - public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull public abstract String getHlsUrl() throws ParsingException; public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 9119cc21b..ffa3028bd 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -126,6 +126,10 @@ public class StreamInfo extends Info { return dashMpdUrl; } + public String getHlsUrl() { + return hlsUrl; + } + public StreamInfoItem getNextVideo() { return next_video; } @@ -206,6 +210,10 @@ public class StreamInfo extends Info { this.dashMpdUrl = dashMpdUrl; } + public void setHlsUrl(String hlsUrl) { + this.hlsUrl = hlsUrl; + } + public void setNextVideo(StreamInfoItem next_video) { this.next_video = next_video; } @@ -298,6 +306,12 @@ public class StreamInfo extends Info { streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e)); } + try { + streamInfo.setHlsUrl(extractor.getHlsUrl()); + } catch (Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e)); + } + /* Load and extract audio */ try { streamInfo.setAudioStreams(extractor.getAudioStreams()); @@ -447,6 +461,7 @@ public class StreamInfo extends Info { // crawling such a file is not service dependent. Therefore getting audio only streams by yust // providing the dash mpd file will be possible in the future. public String dashMpdUrl; + public String hlsUrl; public StreamInfoItem next_video; public List related_streams; From 837dbd6b864931242a021001bc6904cf42051344 Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Sun, 25 Feb 2018 14:31:42 -0800 Subject: [PATCH 3/6] -Fixed live stream info using video stream type. --- .../youtube/YoutubeStreamExtractor.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 38b94bdc6..a3896a678 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -441,7 +441,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull public List getSubtitlesDefault() throws IOException, ExtractionException { - return getSubtitles(SubtitlesFormat.VTT); + return getSubtitles(SubtitlesFormat.TTML); } @Override @@ -457,7 +457,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamType getStreamType() throws ParsingException { - //todo: if implementing livestream support this value should be generated dynamically + assertPageFetched(); + if (playerArgs == null) return StreamType.NONE; + + try { + if (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || + playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty()) { + return StreamType.LIVE_STREAM; + } + } catch (Exception e) { + throw new ParsingException("Could not get hls manifest url", e); + } return StreamType.VIDEO_STREAM; } @@ -547,7 +557,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { doc = Jsoup.parse(pageContent, getCleanUrl()); final String playerUrl; - // TODO: use embedded videos to fetch DASH manifest for all videos // Check if the video is age restricted if (pageContent.contains(" Date: Thu, 1 Mar 2018 16:31:36 -0800 Subject: [PATCH 5/6] -Fixed Youtube page extraction on flagged / offensive content urls. --- .../extractor/services/youtube/YoutubeStreamExtractor.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 91d43b79a..c5e4c5f95 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -538,13 +538,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; + private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; + private volatile String decryptionCode = ""; private String pageHtml = null; - private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{ + private String getPageHtml(Downloader downloader) throws IOException, ExtractionException { + final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS; if (pageHtml == null) { - pageHtml = downloader.download(getCleanUrl()); + pageHtml = downloader.download(verifiedUrl); } return pageHtml; } From cc3f3b8292423e02e750f7c692f2f45298626892 Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Thu, 1 Mar 2018 16:43:47 -0800 Subject: [PATCH 6/6] -Fixed Youtube page extraction on flagged / offensive content urls. -Added test for flagged content url extraction. --- ...utubeStreamExtractorControversialTest.java | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java new file mode 100644 index 000000000..de8fdca6a --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -0,0 +1,125 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +/** + * Test for {@link YoutubeStreamUrlIdHandler} + */ +public class YoutubeStreamExtractorControversialTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); + assertEquals(extractor.getTimeStamp() + "", "174"); + } + + @Test + @Ignore + public void testGetAgeLimit() throws ParsingException { + assertEquals(18, extractor.getAgeLimit()); + } + + @Test + public void testGetName() throws ParsingException { + assertNotNull("name is null", extractor.getName()); + assertFalse("name is empty", extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + @Ignore // Currently there is no way get the length from restricted videos + @Test + public void testGetLength() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + @Test + public void testGetViews() throws ParsingException { + assertTrue(extractor.getViewCount() > 0); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + // FIXME: 25.11.17 Are there no streams or are they not listed? + @Ignore + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + // audio streams are not always necessary + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + List streams = new ArrayList<>(); + streams.addAll(extractor.getVideoStreams()); + streams.addAll(extractor.getVideoOnlyStreams()); + assertTrue(streams.size() > 0); + } + + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(!extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + } +}