diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 42d183f42..96d7d566c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; @@ -63,6 +64,19 @@ public class PeertubeParsingHelper { } public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl) throws ParsingException { + collectStreamsFrom(collector, json, baseUrl, false); + } + + /** + * Collect stream from json with collector + * + * @param collector the collector used to collect information + * @param json the file to retrieve data from + * @param baseUrl the base Url of the instance + * @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor + * @throws ParsingException + */ + public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl, boolean sepia) throws ParsingException { final JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); @@ -73,9 +87,15 @@ public class PeertubeParsingHelper { for (final Object c : contents) { if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + PeertubeStreamInfoItemExtractor extractor; + if (sepia) { + extractor = new PeertubeSepiaStreamInfoItemExtractor(item, baseUrl); + } else { + extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + } collector.commit(extractor); } } } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 77deece79..ea1a78442 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -15,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import java.util.List; + import static java.util.Arrays.asList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; @@ -59,7 +61,12 @@ public class PeertubeService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { - return new PeertubeSearchExtractor(this, queryHandler); + final List contentFilters = queryHandler.getContentFilters(); + boolean external = false; + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) { + external = true; + } + return new PeertubeSearchExtractor(this, queryHandler, external); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 0540c1979..c62cad912 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -30,8 +30,17 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeSearchExtractor extends SearchExtractor { + + // if we should use PeertubeSepiaStreamInfoItemExtractor + private boolean sepia; + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + this(service, linkHandler, false); + } + + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, boolean sepia) { super(service, linkHandler); + this.sepia = sepia; } @Nonnull @@ -79,7 +88,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { final long total = json.getLong("total"); final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - collectStreamsFrom(collector, json, getBaseUrl()); + collectStreamsFrom(collector, json, getBaseUrl(), sepia); return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSepiaStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSepiaStreamInfoItemExtractor.java new file mode 100644 index 000000000..925aa9f0b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSepiaStreamInfoItemExtractor.java @@ -0,0 +1,22 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import com.grack.nanojson.JsonObject; + +/** + * A StreamInfoItem collected from SepiaSearch + */ +public class PeertubeSepiaStreamInfoItemExtractor extends PeertubeStreamInfoItemExtractor { + + public PeertubeSepiaStreamInfoItemExtractor(JsonObject item, String baseUrl) { + super(item, baseUrl); + final String embedUrl = super.item.getString("embedUrl"); + final String embedPath = super.item.getString("embedPath"); + final String itemBaseUrl = embedUrl.replace(embedPath, ""); + setBaseUrl(itemBaseUrl); + + // Usually, all videos, pictures and other content are hosted on the instance, + // or can be accessed by the same URL path if the instance with baseUrl federates the one where the video is actually uploaded + // But it can't be accessed with Sepiasearch, so we use the item's instance as base URL + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index 489f5fe0c..4ab27ae46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -11,8 +10,9 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { + protected final JsonObject item; - private final String baseUrl; + private String baseUrl; public PeertubeStreamInfoItemExtractor(final JsonObject item, final String baseUrl) { this.item = item; @@ -84,4 +84,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor public long getDuration() { return item.getLong("duration"); } + + protected void setBaseUrl(final String baseUrl) { + this.baseUrl = baseUrl; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 8acf3e80b..98c25b0a3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -12,6 +12,8 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public static final String CHARSET_UTF_8 = "UTF-8"; public static final String VIDEOS = "videos"; + public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index + public static final String SEPIA_BASE_URL = "https://sepiasearch.org"; public static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; public static PeertubeSearchQueryHandlerFactory getInstance() { @@ -20,7 +22,12 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { - String baseUrl = ServiceList.PeerTube.getBaseUrl(); + String baseUrl; + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) { + baseUrl = SEPIA_BASE_URL; + } else { + baseUrl = ServiceList.PeerTube.getBaseUrl(); + } return getUrl(searchString, contentFilters, sortFilter, baseUrl); } @@ -38,6 +45,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[]{VIDEOS}; + return new String[]{ + VIDEOS, + SEPIA_VIDEOS + }; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java index 80654a402..e1b2313aa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import javax.annotation.Nullable; @@ -43,6 +44,29 @@ public class PeertubeSearchExtractorTest { @Nullable @Override public String expectedSearchSuggestion() { return null; } } + public static class SepiaSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kde"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = PeerTube.getSearchExtractor(QUERY, singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + public static class PagingTest { @Test public void duplicatedItemsCheck() throws Exception { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index d7bd92aaf..ffc70f1a5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -3,7 +3,9 @@ package org.schabi.newpipe.extractor.services.peertube.search; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; @@ -22,5 +24,7 @@ public class PeertubeSearchQHTest { assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + assertEquals("https://sepiasearch.org/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "").getUrl()); + assertEquals("https://anotherpeertubeindex.com/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "", "https://anotherpeertubeindex.com").getUrl()); } } \ No newline at end of file