From e8e535b815e6a99dee08d1f3c2536868c8c03de0 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 00:25:34 +0100 Subject: [PATCH 1/5] mediaccc: update linkhandlers & refactor tests --- .../extractors/MediaCCCStreamExtractor.java | 5 +- .../MediaCCCConferenceLinkHandlerFactory.java | 46 +++- .../MediaCCCStreamLinkHandlerFactory.java | 41 +++- .../MediaCCCConferenceExtractorTest.java | 91 +++++--- .../MediaCCCStreamExtractorTest.java | 204 ++++++++++++------ 5 files changed, 279 insertions(+), 108 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index c499758d1..424e8fdd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -170,15 +170,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return null; } - @Nonnull @Override public List getSubtitlesDefault() throws IOException, ExtractionException { return null; } - @Nonnull @Override - public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { return null; } @@ -212,7 +210,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } catch (JsonParserException jpe) { throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); } - } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 3c1817674..4d77b0728 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; +import java.net.MalformedURLException; +import java.net.URL; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { @@ -14,19 +18,43 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory } @Override - public String getId(String url) throws ParsingException { - if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { - return url.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if (url.startsWith("https://media.ccc.de/c/")) { - return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); - } else { - throw new ParsingException("Could not get id from url: " + url); + public String getId(String urlString) throws ParsingException { + if (urlString.startsWith("https://api.media.ccc.de/public/conferences/")) { + return urlString.replace("https://api.media.ccc.de/public/conferences/", ""); + } else if (urlString.startsWith("https://media.ccc.de/c/")) { + return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", urlString); } + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (path.contains("b/")) { + return path.substring(2); + } + + throw new ParsingException("Could not get id from url: " + url); + } @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/conferences/") - || url.startsWith("https://media.ccc.de/c/"); + try { + getId(url); + return true; + } catch (FoundAdException fe) { + throw fe; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index bf291a60e..f7262f0c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -1,16 +1,39 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; +import java.net.URL; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { - if (url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q=")) { - return url.replace("https://api.media.ccc.de/public/events/", ""); + public String getId(String urlString) throws ParsingException { + if (urlString.startsWith("https://api.media.ccc.de/public/events/") && + !urlString.contains("?q=")) { + return urlString.replace("https://api.media.ccc.de/public/events/", ""); } + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (path.contains("v/")) { + return path.substring(2); + } + throw new ParsingException("Could not get id from url: " + url); } @@ -21,7 +44,13 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q="); + try { + getId(url); + return true; + } catch (FoundAdException fe) { + throw fe; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 15a68a730..a6e78885d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -4,47 +4,86 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCConferenceExtractor} */ public class MediaCCCConferenceExtractorTest { - private static ChannelExtractor extractor; - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017"); - extractor.fetchPage(); + public static class FrOSCon2017 { + private static MediaCCCConferenceExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017"); + extractor.fetchPage(); + } + + @Test + public void testName() throws Exception { + assertEquals("FrOSCon 2017", extractor.getName()); + } + + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + } + + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertEquals(97, extractor.getInitialPage().getItems().size()); + } } - @Test - public void testName() throws Exception { - assertEquals("FrOSCon 2017", extractor.getName()); - } + public static class Oscal2019 { + private static MediaCCCConferenceExtractor extractor; - @Test - public void testGetUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19"); + extractor.fetchPage(); + } - @Test - public void testGetOriginalUrl() throws Exception { - assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); - } + @Test + public void testName() throws Exception { + assertEquals("Open Source Conference Albania 2019", extractor.getName()); + } - @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); - } + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl()); + } - @Test - public void testGetInitalPage() throws Exception { - assertEquals(97, extractor.getInitialPage().getItems().size()); + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertTrue(extractor.getInitialPage().getItems().size() >= 21); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index ed9dcee71..195656b17 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -1,101 +1,179 @@ package org.schabi.newpipe.extractor.services.media_ccc; +import com.grack.nanojson.JsonObject; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.BaseExtractorTest; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; +import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertFalse; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCStreamExtractor} */ -public class MediaCCCStreamExtractorTest implements BaseExtractorTest { - private static StreamExtractor extractor; +public class MediaCCCStreamExtractorTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); + public static class Gpn18Tmux { + private static MediaCCCStreamExtractor extractor; - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); - extractor.fetchPage(); + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht"); + extractor.fetchPage(); + } + + @Test + public void testServiceId() throws Exception { + assertEquals(2, extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); + } + + @Test + public void testThumbnail() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + } + + @Test + public void testUploaderName() throws Exception { + assertEquals("gpn18", extractor.getUploaderName()); + } + + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + } + + @Test + public void testUploaderAvatarUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + } + + @Test + public void testVideoStreams() throws Exception { + assertEquals(4, extractor.getVideoStreams().size()); + } + + @Test + public void testAudioStreams() throws Exception { + assertEquals(2, extractor.getAudioStreams().size()); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } - @Override - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } + public static class _36c3PrivacyMessaging { + private static MediaCCCStreamExtractor extractor; - @Override - public void testName() throws Exception { - assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging"); + extractor.fetchPage(); + } - @Override - public void testId() throws Exception { - assertEquals("", extractor.getId()); - } + @Test + public void testName() throws Exception { + assertEquals("What's left for private messaging?", extractor.getName()); + } - @Override - public void testUrl() throws Exception { - assertEquals("", extractor.getUrl()); - } + @Test + public void testId() throws Exception { + assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId()); + } - @Override - public void testOriginalUrl() throws Exception { - assertEquals("", extractor.getOriginalUrl()); - } + @Test + public void testUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); + } - @Test - public void testThumbnail() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); - } + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); + } - @Test - public void testUploaderName() throws Exception { - assertEquals("gpn18", extractor.getUploaderName()); - } + @Test + public void testThumbnail() throws Exception { + assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); + } - @Test - public void testUploaderUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); - } + @Test + public void testUploaderName() throws Exception { + assertEquals("36c3", extractor.getUploaderName()); + } - @Test - public void testUploaderAvatarUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); - } + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); + } - @Test - public void testVideoStreams() throws Exception { - assertEquals(4, extractor.getVideoStreams().size()); - } + @Test + public void testUploaderAvatarUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); + } - @Test - public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); - } + @Test + public void testVideoStreams() throws Exception { + assertEquals(8, extractor.getVideoStreams().size()); + } - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); - } + @Test + public void testAudioStreams() throws Exception { + assertEquals(2, extractor.getAudioStreams().size()); + } - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } -} +} \ No newline at end of file From 9701c7d800ae91fda88791dc21e02ee80888c030 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 18:59:46 +0100 Subject: [PATCH 2/5] invidious shortened links --- .../linkHandler/YoutubeStreamLinkHandlerFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index e3ac4d520..15e998945 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -190,7 +190,12 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } - break; + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsID(viewQueryValue); + } + + return assertIsID(path); } } From f742a6bd3ea68a1d243aeceae5092703e92c3cc9 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 21:24:50 +0100 Subject: [PATCH 3/5] code optimization --- .../MediaCCCConferenceLinkHandlerFactory.java | 37 ++++--------------- .../MediaCCCStreamLinkHandlerFactory.java | 4 +- .../MediaCCCConferenceExtractorTest.java | 2 +- .../MediaCCCStreamExtractorTest.java | 4 -- 4 files changed, 9 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 4d77b0728..f4074c0d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -1,13 +1,9 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; -import java.net.MalformedURLException; -import java.net.URL; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { @@ -18,32 +14,15 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory } @Override - public String getId(String urlString) throws ParsingException { - if (urlString.startsWith("https://api.media.ccc.de/public/conferences/")) { - return urlString.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if (urlString.startsWith("https://media.ccc.de/c/")) { - return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", urlString); + public String getId(String url) throws ParsingException { + if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { + return url.replace("https://api.media.ccc.de/public/conferences/", ""); + } else if (url.startsWith("https://media.ccc.de/c/")) { + return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); + } else if (url.startsWith("https://media.ccc.de/b/")) { + return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url); } - - URL url; - try { - url = Utils.stringToURL(urlString); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("The given URL is not valid"); - } - - String path = url.getPath(); - // remove leading "/" of URL-path if URL-path is given - if (!path.isEmpty()) { - path = path.substring(1); - } - - if (path.contains("b/")) { - return path.substring(2); - } - throw new ParsingException("Could not get id from url: " + url); - } @Override @@ -51,8 +30,6 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory try { getId(url); return true; - } catch (FoundAdException fe) { - throw fe; } catch (ParsingException e) { return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index f7262f0c3..76e5df895 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -30,7 +30,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { path = path.substring(1); } - if (path.contains("v/")) { + if (path.startsWith("v/")) { return path.substring(2); } @@ -47,8 +47,6 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { try { getId(url); return true; - } catch (FoundAdException fe) { - throw fe; } catch (ParsingException e) { return false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index a6e78885d..258a55b00 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 195656b17..9cb856625 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -1,23 +1,19 @@ package org.schabi.newpipe.extractor.services.media_ccc; -import com.grack.nanojson.JsonObject; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; -import static org.junit.Assert.assertFalse; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** From 0cd5e05b7b927a3d53048e5347a4f37fd8920efa Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 2 Mar 2020 22:38:44 +0100 Subject: [PATCH 4/5] MediaCCCLH: use substring instead of replace; improve a bit tests, return Collections.emptyList(); instead of null where it's annotated @NonNull --- .../extractors/MediaCCCStreamExtractor.java | 11 +++++++---- .../MediaCCCStreamLinkHandlerFactory.java | 2 +- .../media_ccc/MediaCCCStreamExtractorTest.java | 12 ++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 424e8fdd1..59bfe1f4f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -104,13 +105,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public String getDashMpdUrl() throws ParsingException { - return null; + return ""; } @Nonnull @Override public String getHlsUrl() throws ParsingException { - return null; + return ""; } @Override @@ -170,14 +171,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return null; } + @Nonnull @Override public List getSubtitlesDefault() throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } + @Nonnull @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index 76e5df895..2c91d0564 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -14,7 +14,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { public String getId(String urlString) throws ParsingException { if (urlString.startsWith("https://api.media.ccc.de/public/events/") && !urlString.contains("?q=")) { - return urlString.replace("https://api.media.ccc.de/public/events/", ""); + return urlString.substring(39); //remove api…/public/events part } URL url; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 9cb856625..c0f51437d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -14,6 +15,7 @@ import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -49,16 +51,19 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); } @Test public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); } @Test public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); } @@ -69,11 +74,13 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); } @Test public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); } @@ -122,16 +129,19 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); } @Test public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); } @Test public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); } @@ -142,11 +152,13 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); } @Test public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); } From 66518ec44452f2fd3916805a07d3959409b0e16c Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 13 Mar 2020 20:07:36 +0100 Subject: [PATCH 5/5] check wether mediaccc streams url are secure --- .../MediaCCCStreamExtractorTest.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index c0f51437d..8120e07a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -7,11 +7,14 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.List; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; @@ -86,12 +89,20 @@ public class MediaCCCStreamExtractorTest { @Test public void testVideoStreams() throws Exception { - assertEquals(4, extractor.getVideoStreams().size()); + List videoStreamList = extractor.getVideoStreams(); + assertEquals(4, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test @@ -164,12 +175,20 @@ public class MediaCCCStreamExtractorTest { @Test public void testVideoStreams() throws Exception { - assertEquals(8, extractor.getVideoStreams().size()); + List videoStreamList = extractor.getVideoStreams(); + assertEquals(8, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test