be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #256 from B0pol/mediaccc 

Support for non-api mediaccc links, invidious shortened links, refactor mediaccc tests
This commit is contained in:
Tobias Groza 2020-03-15 17:04:09 +01:00 committed by GitHub
parent 40b3bc138f 66518ec444
commit 65a7eda446
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 287 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
@ -104,13 +105,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public String getDashMpdUrl() throws ParsingException { public String getDashMpdUrl() throws ParsingException {
return null; return "";
} }
@Nonnull @Nonnull
@Override @Override
public String getHlsUrl() throws ParsingException { public String getHlsUrl() throws ParsingException {
return null; return "";
} }
@Override @Override
@ -173,13 +174,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
return null; return Collections.emptyList();
} }
@Nonnull @Nonnull
@Override @Override
public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
return null; return Collections.emptyList();
} }
@Override @Override
@ -212,7 +213,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
} catch (JsonParserException jpe) { } catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe);
} }
} }
@Nonnull @Nonnull

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java
View File

@ -19,14 +19,19 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
return url.replace("https://api.media.ccc.de/public/conferences/", ""); return url.replace("https://api.media.ccc.de/public/conferences/", "");
} else if (url.startsWith("https://media.ccc.de/c/")) { } else if (url.startsWith("https://media.ccc.de/c/")) {
return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
} else { } else if (url.startsWith("https://media.ccc.de/b/")) {
throw new ParsingException("Could not get id from url: " + url); return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
} }
throw new ParsingException("Could not get id from url: " + url);
} }
@Override @Override
public boolean onAcceptUrl(String url) throws ParsingException { public boolean onAcceptUrl(String url) throws ParsingException {
return url.startsWith("https://api.media.ccc.de/public/conferences/") try {
|| url.startsWith("https://media.ccc.de/c/"); getId(url);
return true;
} catch (ParsingException e) {
return false;
}
} }
} }

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java
View File

@ -1,16 +1,39 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
import java.net.URL;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
@Override @Override
public String getId(String url) throws ParsingException { public String getId(String urlString) throws ParsingException {
if (url.startsWith("https://api.media.ccc.de/public/events/") && if (urlString.startsWith("https://api.media.ccc.de/public/events/") &&
!url.contains("?q=")) { !urlString.contains("?q=")) {
return url.replace("https://api.media.ccc.de/public/events/", ""); return urlString.substring(39); //remove api…/public/events part
} }
URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String path = url.getPath();
// remove leading "/" of URL-path if URL-path is given
if (!path.isEmpty()) {
path = path.substring(1);
}
if (path.startsWith("v/")) {
return path.substring(2);
}
throw new ParsingException("Could not get id from url: " + url); throw new ParsingException("Could not get id from url: " + url);
} }
@ -21,7 +44,11 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
@Override @Override
public boolean onAcceptUrl(String url) throws ParsingException { public boolean onAcceptUrl(String url) throws ParsingException {
return url.startsWith("https://api.media.ccc.de/public/events/") && try {
!url.contains("?q="); getId(url);
return true;
} catch (ParsingException e) {
return false;
}
} }
} }

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View File

@ -202,7 +202,12 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsID(id); return assertIsID(id);
} }
break; String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsID(viewQueryValue);
}
return assertIsID(path);
} }
} }

91
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View File

@ -4,47 +4,86 @@ import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/** /**
* Test {@link MediaCCCConferenceExtractor} * Test {@link MediaCCCConferenceExtractor}
*/ */
public class MediaCCCConferenceExtractorTest { public class MediaCCCConferenceExtractorTest {
private static ChannelExtractor extractor;
@BeforeClass public static class FrOSCon2017 {
public static void setUpClass() throws Exception { private static MediaCCCConferenceExtractor extractor;
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017"); @BeforeClass
extractor.fetchPage(); public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017");
extractor.fetchPage();
}
@Test
public void testName() throws Exception {
assertEquals("FrOSCon 2017", extractor.getName());
}
@Test
public void testGetUrl() throws Exception {
assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl());
}
@Test
public void testGetOriginalUrl() throws Exception {
assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl());
}
@Test
public void testGetThumbnailUrl() throws Exception {
assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl());
}
@Test
public void testGetInitalPage() throws Exception {
assertEquals(97, extractor.getInitialPage().getItems().size());
}
} }
@Test public static class Oscal2019 {
public void testName() throws Exception { private static MediaCCCConferenceExtractor extractor;
assertEquals("FrOSCon 2017", extractor.getName());
}
@Test @BeforeClass
public void testGetUrl() throws Exception { public static void setUpClass() throws Exception {
assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); NewPipe.init(DownloaderTestImpl.getInstance());
} extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19");
extractor.fetchPage();
}
@Test @Test
public void testGetOriginalUrl() throws Exception { public void testName() throws Exception {
assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); assertEquals("Open Source Conference Albania 2019", extractor.getName());
} }
@Test @Test
public void testGetThumbnailUrl() throws Exception { public void testGetUrl() throws Exception {
assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl());
} }
@Test @Test
public void testGetInitalPage() throws Exception { public void testGetOriginalUrl() throws Exception {
assertEquals(97, extractor.getInitialPage().getItems().size()); assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl());
}
@Test
public void testGetThumbnailUrl() throws Exception {
assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl());
}
@Test
public void testGetInitalPage() throws Exception {
assertTrue(extractor.getInitialPage().getItems().size() >= 21);
}
} }
} }

229
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -6,96 +6,201 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseExtractorTest;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.UtilsTest;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.List;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/** /**
* Test {@link MediaCCCStreamExtractor} * Test {@link MediaCCCStreamExtractor}
*/ */
public class MediaCCCStreamExtractorTest implements BaseExtractorTest { public class MediaCCCStreamExtractorTest {
private static StreamExtractor extractor;
@BeforeClass public static class Gpn18Tmux {
public static void setUpClass() throws Exception { private static MediaCCCStreamExtractor extractor;
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); @BeforeClass
extractor.fetchPage(); public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht");
extractor.fetchPage();
}
@Test
public void testServiceId() throws Exception {
assertEquals(2, extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName());
}
@Test
public void testId() throws Exception {
assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId());
}
@Test
public void testUrl() throws Exception {
assertIsSecureUrl(extractor.getUrl());
assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws Exception {
assertIsSecureUrl(extractor.getOriginalUrl());
assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl());
}
@Test
public void testThumbnail() throws Exception {
assertIsSecureUrl(extractor.getThumbnailUrl());
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
}
@Test
public void testUploaderName() throws Exception {
assertEquals("gpn18", extractor.getUploaderName());
}
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
}
@Test
public void testUploaderAvatarUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl());
}
@Test
public void testVideoStreams() throws Exception {
List<VideoStream> videoStreamList = extractor.getVideoStreams();
assertEquals(4, videoStreamList.size());
for (VideoStream stream : videoStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test
public void testAudioStreams() throws Exception {
List<AudioStream> audioStreamList = extractor.getAudioStreams();
assertEquals(2, audioStreamList.size());
for (AudioStream stream : audioStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
} }
@Override public static class _36c3PrivacyMessaging {
public void testServiceId() throws Exception { private static MediaCCCStreamExtractor extractor;
assertEquals(2, extractor.getServiceId());
}
@Override @BeforeClass
public void testName() throws Exception { public static void setUpClass() throws Exception {
assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); NewPipe.init(DownloaderTestImpl.getInstance());
} extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging");
extractor.fetchPage();
}
@Override @Test
public void testId() throws Exception { public void testName() throws Exception {
assertEquals("", extractor.getId()); assertEquals("What's left for private messaging?", extractor.getName());
} }
@Override @Test
public void testUrl() throws Exception { public void testId() throws Exception {
assertEquals("", extractor.getUrl()); assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId());
} }
@Override @Test
public void testOriginalUrl() throws Exception { public void testUrl() throws Exception {
assertEquals("", extractor.getOriginalUrl()); assertIsSecureUrl(extractor.getUrl());
} assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
}
@Test @Test
public void testThumbnail() throws Exception { public void testOriginalUrl() throws Exception {
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); assertIsSecureUrl(extractor.getOriginalUrl());
} assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl());
}
@Test @Test
public void testUploaderName() throws Exception { public void testThumbnail() throws Exception {
assertEquals("gpn18", extractor.getUploaderName()); assertIsSecureUrl(extractor.getThumbnailUrl());
} assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
}
@Test @Test
public void testUploaderUrl() throws Exception { public void testUploaderName() throws Exception {
assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); assertEquals("36c3", extractor.getUploaderName());
} }
@Test @Test
public void testUploaderAvatarUrl() throws Exception { public void testUploaderUrl() throws Exception {
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); assertIsSecureUrl(extractor.getUploaderUrl());
} assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
}
@Test @Test
public void testVideoStreams() throws Exception { public void testUploaderAvatarUrl() throws Exception {
assertEquals(4, extractor.getVideoStreams().size()); assertIsSecureUrl(extractor.getUploaderAvatarUrl());
} assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl());
}
@Test @Test
public void testAudioStreams() throws Exception { public void testVideoStreams() throws Exception {
assertEquals(2, extractor.getAudioStreams().size()); List<VideoStream> videoStreamList = extractor.getVideoStreams();
} assertEquals(8, videoStreamList.size());
for (VideoStream stream : videoStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test @Test
public void testGetTextualUploadDate() throws ParsingException { public void testAudioStreams() throws Exception {
Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); List<AudioStream> audioStreamList = extractor.getAudioStreams();
} assertEquals(2, audioStreamList.size());
for (AudioStream stream : audioStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test @Test
public void testGetUploadDate() throws ParsingException, ParseException { public void testGetTextualUploadDate() throws ParsingException {
final Calendar instance = Calendar.getInstance(); Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate());
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); }
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
} }
} }