From 150345929202a0aae40633bf64d0d25a9c2cfae4 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 28 Jan 2019 12:18:16 +0100 Subject: [PATCH] add kiosk for conferences --- .../extractor/kiosk/KioskExtractor.java | 3 +- .../newpipe/extractor/kiosk/KioskList.java | 5 ++ .../services/media_ccc/MediaCCCService.java | 17 ++++- .../extractors/MediaCCCConferenceKiosk.java | 69 +++++++++++++++++++ .../extractors/MediaCCCSearchExtractor.java | 4 +- .../MediaCCCConferenceInfoItemExtractor.java | 44 ++++++++++++ ...aCCCConferencesListLinkHandlerFactory.java | 25 +++++++ .../soundcloud/SoundcloudChartsExtractor.java | 2 +- .../extractors/YoutubeTrendingExtractor.java | 2 +- .../MediaCCCConferenceListExtractorTest.java | 59 ++++++++++++++++ 10 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 481bb63ca..e2786bbac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.kiosk; * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -29,7 +30,7 @@ import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nonnull; -public abstract class KioskExtractor extends ListExtractor { +public abstract class KioskExtractor extends ListExtractor { private final String id; public KioskExtractor(StreamingService streamingService, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 22a9f2ca8..3636110d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -49,6 +49,11 @@ public class KioskList { defaultKiosk = kioskType; } + public KioskExtractor getDefaultKioskExtractor() + throws ExtractionException, IOException { + return getDefaultKioskExtractor(""); + } + public KioskExtractor getDefaultKioskExtractor(String nextPageUrl) throws ExtractionException, IOException { return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index cab5936d7..4dbf8e7f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -4,20 +4,25 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; +import java.io.IOException; + import static java.util.Arrays.asList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; @@ -77,7 +82,17 @@ public class MediaCCCService extends StreamingService { // add kiosks here e.g.: try { - // Add kiosk here + list.addKioskEntry(new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String kioskId, + Localization localization) throws ExtractionException, IOException { + return new MediaCCCConferenceKiosk(MediaCCCService.this, + new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId, localization); + } + }, new MediaCCCConferencesListLinkHandlerFactory(), "conferences"); + list.setDefaultKiosk("conferences"); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java new file mode 100644 index 000000000..7eac10c11 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -0,0 +1,69 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class MediaCCCConferenceKiosk extends KioskExtractor { + + private JsonObject doc; + + public MediaCCCConferenceKiosk(StreamingService streamingService, + ListLinkHandler linkHandler, + String kioskId, + Localization localization) { + super(streamingService, linkHandler, kioskId, localization); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + JsonArray conferences = doc.getArray("conferences"); + ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); + for(int i = 0; i < conferences.size(); i++) { + collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); + } + + return new InfoItemsPage<>(collector, ""); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return ""; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String site = downloader.download(getLinkHandler().getUrl()); + try { + doc = JsonParser.object().from(site); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json.", jpe); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return doc.getString("Conferences"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 6d9b2cbff..4e1a4f1c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -20,7 +20,7 @@ import java.io.IOException; public class MediaCCCSearchExtractor extends SearchExtractor { - JsonObject doc; + private JsonObject doc; public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) { super(service, linkHandler, localization); @@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - return null; + return InfoItemsPage.emptyPage(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java new file mode 100644 index 000000000..05914dd75 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -0,0 +1,44 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor { + + JsonObject conference; + + public MediaCCCConferenceInfoItemExtractor(JsonObject conference) { + this.conference = conference; + } + + @Override + public String getDescription() throws ParsingException { + return ""; + } + + @Override + public long getSubscriberCount() throws ParsingException { + return -1; + } + + @Override + public long getStreamCount() throws ParsingException { + return -1; + } + + @Override + public String getName() throws ParsingException { + return conference.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return conference.getString("url"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return conference.getString("logo_url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java new file mode 100644 index 000000000..603f1ea2d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory { + @Override + public String getId(String url) throws ParsingException { + return "conferences"; + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return "https://api.media.ccc.de/public/conferences"; + } + + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + return url.equals("https://media.ccc.de/b/conferences") + || url.equals("https://api.media.ccc.de/public/conferences"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 8f02b1c68..1a02166e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -14,7 +14,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -public class SoundcloudChartsExtractor extends KioskExtractor { +public class SoundcloudChartsExtractor extends KioskExtractor { private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index f33f7b8a5..b065aa630 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -37,7 +37,7 @@ import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nonnull; import java.io.IOException; -public class YoutubeTrendingExtractor extends KioskExtractor { +public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java new file mode 100644 index 000000000..2116916f2 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.media_ccc; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.util.List; + +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; +import static org.junit.Assert.assertTrue; + + +/** + * Test {@link MediaCCCConferenceKiosk} + */ +public class MediaCCCConferenceListExtractorTest { + + private static KioskExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB")); + extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); + extractor.fetchPage(); + } + + @Test + public void getConferencesListTest() throws Exception { + assertTrue("returned list was to small", + extractor.getInitialPage().getItems().size() >= 174); + } + + @Test + public void conferenceTypeTest() throws Exception { + assertTrue(contains(extractor.getInitialPage().getItems(), "FrOSCon 2016")); + assertTrue(contains(extractor.getInitialPage().getItems(), "ChaosWest @ 35c3")); + assertTrue(contains(extractor.getInitialPage().getItems(), "CTreffOS chaOStalks")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Datenspuren 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Chaos Singularity 2017")); + assertTrue(contains(extractor.getInitialPage().getItems(), "SIGINT10")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Vintage Computing Festival Berlin 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "FIfFKon 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "33C3: trailers")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Blinkenlights")); + } + + private boolean contains(List itemList, String name) { + for(InfoItem item : itemList) { + if(item.getName().equals(name)) + return true; + } + return false; + } +}