From 4de99ae28f28c52eab0584c62c93f7de25fe4f92 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 6 Nov 2018 14:24:47 +0100 Subject: [PATCH] add support for segmented streams --- .../newpipe/extractor/stream/StreamInfo.java | 32 +++++++ .../extractor/utils/DashMpdParser.java | 91 +++++++++++++++---- .../YoutubeStreamExtractorDASHTest.java | 10 +- .../search/YoutubeSearchCountTest.java | 2 +- 4 files changed, 115 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 5ab2c2e83..65a024983 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -152,6 +152,9 @@ public class StreamInfo extends Info { streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); streamInfo.getAudioStreams().addAll(result.getAudioStreams()); streamInfo.getVideoStreams().addAll(result.getVideoStreams()); + streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams(); + streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams(); + streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams(); } catch (Exception e) { // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl), // just skip the exception (but store it somewhere), as we later check if we have streams anyway. @@ -271,6 +274,11 @@ public class StreamInfo extends Info { private List videoOnlyStreams; private String dashMpdUrl; + private List segmentedVideoStreams; + private List segmentedAudioStreams; + private List segmentedVideoOnlyStreams; + + private String hlsUrl; private StreamInfoItem nextVideo; private List relatedStreams; @@ -431,6 +439,30 @@ public class StreamInfo extends Info { this.dashMpdUrl = dashMpdUrl; } + public List getSegmentedVideoStreams() { + return segmentedVideoStreams; + } + + public void setSegmentedVideoStreams(List segmentedVideoStreams) { + this.segmentedVideoStreams = segmentedVideoStreams; + } + + public List getSegmentedAudioStreams() { + return segmentedAudioStreams; + } + + public void setSegmentedAudioStreams(List segmentedAudioStreams) { + this.segmentedAudioStreams = segmentedAudioStreams; + } + + public List getSegmentedVideoOnlyStreams() { + return segmentedVideoOnlyStreams; + } + + public void setSegmentedVideoOnlyStreams(List segmentedVideoOnlyStreams) { + this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; + } + public String getHlsUrl() { return hlsUrl; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 6c656d772..8a994cbde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -59,10 +59,23 @@ public class DashMpdParser { private final List audioStreams; private final List videoOnlyStreams; - public ParserResult(List videoStreams, List audioStreams, List videoOnlyStreams) { + private final List segmentedVideoStreams; + private final List segmentedAudioStreams; + private final List segmentedVideoOnlyStreams; + + + public ParserResult(List videoStreams, + List audioStreams, + List videoOnlyStreams, + List segmentedVideoStreams, + List segmentedAudioStreams, + List segmentedVideoOnlyStreams) { this.videoStreams = videoStreams; this.audioStreams = audioStreams; this.videoOnlyStreams = videoOnlyStreams; + this.segmentedVideoStreams = segmentedVideoStreams; + this.segmentedAudioStreams = segmentedAudioStreams; + this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; } public List getVideoStreams() { @@ -76,10 +89,22 @@ public class DashMpdParser { public List getVideoOnlyStreams() { return videoOnlyStreams; } + + public List getSegmentedVideoStreams() { + return segmentedVideoStreams; + } + + public List getSegmentedAudioStreams() { + return segmentedAudioStreams; + } + + public List getSegmentedVideoOnlyStreams() { + return segmentedVideoOnlyStreams; + } } /** - * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest, + * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest, * then it will search for any stream that the ItagItem has (by the id). *

* It has video, video only and audio streams and will only add to the list if it don't @@ -90,7 +115,8 @@ public class DashMpdParser { * * @param streamInfo where the parsed streams will be added */ - public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { + public static ParserResult getStreams(final StreamInfo streamInfo) + throws DashMpdParsingException, ReCaptchaException { String dashDoc; Downloader downloader = NewPipe.getDownloader(); try { @@ -113,6 +139,10 @@ public class DashMpdParser { final List audioStreams = new ArrayList<>(); final List videoOnlyStreams = new ArrayList<>(); + final List segmentedVideoStreams = new ArrayList<>(); + final List segmentedAudioStreams = new ArrayList<>(); + final List segmentedVideoOnlyStreams = new ArrayList<>(); + for (int i = 0; i < representationList.getLength(); i++) { final Element representation = (Element) representationList.item(i); try { @@ -126,34 +156,61 @@ public class DashMpdParser { // instead we need to add the "media=" value from the tags inside the // tag in order to get a full working url. However each of these is just pointing to a part of the // video, so we can not return a URL with a working stream here. - // We decided not to ignore such streams for the moment. - if (itag != null && segmentationList == null) { + // Instead of putting those streams into the list of regular stream urls wie put them in a + // for example "segmentedVideoStreams" list. + if (itag != null) { final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); - - if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { - audioStreams.add(audioStream); + if(segmentationList == null) { + final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); + if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { + audioStreams.add(audioStream); + } + } else { + segmentedAudioStreams.add( + new AudioStream(id, mediaFormat, itag.avgBitrate)); } } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly); - if (isVideoOnly) { - if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { - streamInfo.getVideoOnlyStreams().add(videoStream); - videoOnlyStreams.add(videoStream); + if(segmentationList == null) { + final VideoStream videoStream = new VideoStream(url, + mediaFormat, + itag.resolutionString, + isVideoOnly); + + if (isVideoOnly) { + if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { + videoOnlyStreams.add(videoStream); + } + } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { + videoStreams.add(videoStream); + } + } else { + final VideoStream videoStream = new VideoStream(id, + mediaFormat, + itag.resolutionString, + isVideoOnly); + + if(isVideoOnly) { + segmentedVideoOnlyStreams.add(videoStream); + } else { + segmentedVideoStreams.add(videoStream); } - } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { - videoStreams.add(videoStream); } } } } catch (Exception ignored) { } } - return new ParserResult(videoStreams, audioStreams, videoOnlyStreams); + return new ParserResult( + videoStreams, + audioStreams, + videoOnlyStreams, + segmentedVideoStreams, + segmentedAudioStreams, + segmentedVideoOnlyStreams); } catch (Exception e) { throw new DashMpdParsingException("Could not parse Dash mpd", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java index 0d4013e9b..ceba6092d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java @@ -47,15 +47,21 @@ public class YoutubeStreamExtractorDASHTest { @Test public void testGetDashMpd() { - System.out.println(info.getDashMpdUrl()); assertTrue(info.getDashMpdUrl(), info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty()); } @Test - public void testDashMpdParser() { + public void testRegularStreams() { assertEquals(0, info.getAudioStreams().size()); assertEquals(0, info.getVideoOnlyStreams().size()); assertEquals(4, info.getVideoStreams().size()); } + + @Test + public void testSegmentedStreams() { + assertEquals(2, info.getSegmentedAudioStreams().size()); + assertEquals(3, info.getSegmentedVideoOnlyStreams().size()); + assertEquals(0, info.getSegmentedVideoStreams().size()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java index af314e632..92349c85e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java @@ -28,7 +28,7 @@ public class YoutubeSearchCountTest { public void testViewCount() { ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()), - 65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316); + 69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316); } } }