From 1acc53b8dbb609845405f87029f9fcbdd8cf92e2 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 15 May 2021 12:39:23 +0200 Subject: [PATCH] [PeerTube] Add support for HLS streams --- .../extractors/PeertubeStreamExtractor.java | 34 +++++++++-- .../peertube/PeertubeStreamExtractorTest.java | 60 ++++++++++++++++--- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 28dd292a2..46dc63791 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -211,12 +211,37 @@ public class PeertubeStreamExtractor extends StreamExtractor { public List getVideoStreams() throws ExtractionException { assertPageFetched(); final List videoStreams = new ArrayList<>(); + // mp4 try { - final JsonArray streams = json.getArray("files"); + videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files"))); + } catch (Exception ignored) { } + // HLS + try { + final JsonArray streamingPlaylists = json.getArray("streamingPlaylists"); + for (final Object p : streamingPlaylists) { + if (!(p instanceof JsonObject)) continue; + final JsonObject playlist = (JsonObject) p; + videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files"))); + } + } catch (Exception e) { + throw new ParsingException("Could not get video streams", e); + } + + return videoStreams; + } + + private List getVideoStreamsFromArray(final JsonArray streams) throws ParsingException { + try { + final List videoStreams = new ArrayList<>(); for (final Object s : streams) { if (!(s instanceof JsonObject)) continue; final JsonObject stream = (JsonObject) s; - final String url = JsonUtils.getString(stream, "fileUrl"); + final String url; + if (stream.has("fileDownloadUrl")) { + url = JsonUtils.getString(stream, "fileDownloadUrl"); + } else { + url = JsonUtils.getString(stream, "fileUrl"); + } final String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); final String resolution = JsonUtils.getString(stream, "resolution.label"); final String extension = url.substring(url.lastIndexOf(".") + 1); @@ -226,14 +251,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { videoStreams.add(videoStream); } } + return videoStreams; } catch (Exception e) { - throw new ParsingException("Could not get video streams", e); + throw new ParsingException("Could not get video streams from array"); } - return videoStreams; } - @Override public List getVideoOnlyStreams() { return Collections.emptyList(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index f6d040118..76d411065 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -22,10 +22,13 @@ import javax.annotation.Nullable; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -public class PeertubeStreamExtractorTest { +public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractorTest { private static final String BASE_URL = "/videos/watch/"; - public static class WhatIsPeertube extends DefaultStreamExtractorTest { + @Override public boolean expectedHasAudioStreams() { return false; } + @Override public boolean expectedHasFrames() { return false; } + + public static class WhatIsPeertube extends PeertubeStreamExtractorTest { private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d"; private static final String INSTANCE = "https://framatube.org"; private static final int TIMESTAMP_MINUTE = 1; @@ -85,18 +88,60 @@ public class PeertubeStreamExtractorTest { @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; } @Override public long expectedLikeCountAtLeast() { return 120; } @Override public long expectedDislikeCountAtLeast() { return 0; } - @Override public boolean expectedHasAudioStreams() { return false; } - @Override public boolean expectedHasFrames() { return false; } @Override public String expectedHost() { return "framatube.org"; } @Override public String expectedCategory() { return "Science & Technology"; } @Override public String expectedLicence() { return "Attribution - Share Alike"; } @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); } @Override public List expectedTags() { return Arrays.asList("framasoft", "peertube"); } - @Override public int expectedStreamSegmentsCount() { return 0; } + } + + public static class HlsOnlyStreams extends PeertubeStreamExtractorTest { + private static final String ID = "41342cb4-6fa8-402d-a116-1f63a7f438a3"; + private static final String INSTANCE = "https://tilvids.com"; + + private static final String URL = INSTANCE + BASE_URL + ID; + private static StreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel (!) + PeerTube.setInstance(new PeertubeInstance(INSTANCE, "TILvids")); + extractor = PeerTube.getStreamExtractor(URL); + extractor.fetchPage(); + } + + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return "A Goodbye to Flash Games"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } + + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Marinauts"; } + @Override public String expectedUploaderUrl() { return "https://tilvids.com/accounts/marinauts@tilvids.com"; } + @Override public String expectedSubChannelName() { return "Main marinauts channel"; } + @Override public String expectedSubChannelUrl() { return "https://tilvids.com/video-channels/marinauts_channel"; } + @Override public List expectedDescriptionContains() { // CRLF line ending + return Arrays.asList("Goodbye", "Flash Games", "Anthony takes a minute", "Songs used:"); + } + @Override public long expectedLength() { return 362; } + @Override public long expectedViewCountAtLeast() { return 20; } + @Nullable @Override public String expectedUploadDate() { return "2021-04-08 20:15:32.434"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2021-04-08T20:15:32.434Z"; } + @Override public long expectedLikeCountAtLeast() { return 6; } + @Override public long expectedDislikeCountAtLeast() { return 0; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public String expectedHost() { return "tilvids.com"; } + @Override public String expectedCategory() { return "Entertainment"; } + @Override public String expectedLicence() { return "Unknown"; } + @Override public Locale expectedLanguageInfo() { return null; } + @Override public List expectedTags() { return Arrays.asList("Marinauts", "adobe flash", "adobe flash player", "flash games", "the marinauts"); } } @Ignore("Test broken, SSL problem") - public static class AgeRestricted extends DefaultStreamExtractorTest { + public static class AgeRestricted extends PeertubeStreamExtractorTest { private static final String ID = "dbd8e5e1-c527-49b6-b70c-89101dbb9c08"; private static final String INSTANCE = "https://nocensoring.net"; private static final String URL = INSTANCE + "/videos/embed/" + ID; @@ -134,9 +179,6 @@ public class PeertubeStreamExtractorTest { @Override public long expectedLikeCountAtLeast() { return 1; } @Override public long expectedDislikeCountAtLeast() { return 0; } @Override public int expectedAgeLimit() { return 18; } - @Override public boolean expectedHasAudioStreams() { return false; } - @Override public boolean expectedHasSubtitles() { return false; } - @Override public boolean expectedHasFrames() { return false; } @Override public String expectedHost() { return "nocensoring.net"; } @Override public String expectedCategory() { return "Art"; } @Override public String expectedLicence() { return "Attribution"; }