Merge pull request #495 from B0pol/sepiasearch

[PeerTube] Implement sepia search

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.InfoItemsCollector;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -63,6 +64,19 @@ public class PeertubeParsingHelper {
     }
 
     public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl) throws ParsingException {
+        collectStreamsFrom(collector, json, baseUrl, false);
+    }
+
+    /**
+     * Collect stream from json with collector
+     *
+     * @param collector the collector used to collect information
+     * @param json      the file to retrieve data from
+     * @param baseUrl   the base Url of the instance
+     * @param sepia     if we should use PeertubeSepiaStreamInfoItemExtractor
+     * @throws ParsingException
+     */
+    public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl, boolean sepia) throws ParsingException {
         final JsonArray contents;
         try {
             contents = (JsonArray) JsonUtils.getValue(json, "data");
@@ -73,9 +87,15 @@ public class PeertubeParsingHelper {
         for (final Object c : contents) {
             if (c instanceof JsonObject) {
                 final JsonObject item = (JsonObject) c;
-                final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
+                PeertubeStreamInfoItemExtractor extractor;
+                if (sepia) {
+                    extractor = new PeertubeSepiaStreamInfoItemExtractor(item, baseUrl);
+                } else {
+                    extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
+                }
                 collector.commit(extractor);
             }
         }
     }
+
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java

@@ -15,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import java.util.List;
+
 import static java.util.Arrays.asList;
 import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
 import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
@@ -59,7 +61,12 @@ public class PeertubeService extends StreamingService {
 
     @Override
     public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
-        return new PeertubeSearchExtractor(this, queryHandler);
+        final List<String> contentFilters = queryHandler.getContentFilters();
+        boolean external = false;
+        if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
+            external = true;
+        }
+        return new PeertubeSearchExtractor(this, queryHandler, external);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java

@@ -30,8 +30,17 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class PeertubeSearchExtractor extends SearchExtractor {
+
+    // if we should use PeertubeSepiaStreamInfoItemExtractor
+    private boolean sepia;
+
     public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
+        this(service, linkHandler, false);
+    }
+
+    public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, boolean sepia) {
         super(service, linkHandler);
+        this.sepia = sepia;
     }
 
     @Nonnull
@@ -79,7 +88,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
             final long total = json.getLong("total");
 
             final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
-            collectStreamsFrom(collector, json, getBaseUrl());
+            collectStreamsFrom(collector, json, getBaseUrl(), sepia);
 
             return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
         } else {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSepiaStreamInfoItemExtractor.java

@@ -0,0 +1,22 @@
+package org.schabi.newpipe.extractor.services.peertube.extractors;
+
+import com.grack.nanojson.JsonObject;
+
+/**
+ * A StreamInfoItem collected from SepiaSearch
+ */
+public class PeertubeSepiaStreamInfoItemExtractor extends PeertubeStreamInfoItemExtractor {
+
+    public PeertubeSepiaStreamInfoItemExtractor(JsonObject item, String baseUrl) {
+        super(item, baseUrl);
+        final String embedUrl = super.item.getString("embedUrl");
+        final String embedPath = super.item.getString("embedPath");
+        final String itemBaseUrl = embedUrl.replace(embedPath, "");
+        setBaseUrl(itemBaseUrl);
+
+        // Usually, all videos, pictures and other content are hosted on the instance,
+        // or can be accessed by the same URL path if the instance with baseUrl federates the one where the video is actually uploaded
+        // But it can't be accessed with Sepiasearch, so we use the item's instance as base URL
+    }
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -11,8 +10,9 @@ import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
+
     protected final JsonObject item;
-    private final String baseUrl;
+    private String baseUrl;
 
     public PeertubeStreamInfoItemExtractor(final JsonObject item, final String baseUrl) {
         this.item = item;
@@ -84,4 +84,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
     public long getDuration() {
         return item.getLong("duration");
     }
+
+    protected void setBaseUrl(final String baseUrl) {
+        this.baseUrl = baseUrl;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java

@@ -12,6 +12,8 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
 
     public static final String CHARSET_UTF_8 = "UTF-8";
     public static final String VIDEOS = "videos";
+    public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
+    public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
     public static final String SEARCH_ENDPOINT = "/api/v1/search/videos";
 
     public static PeertubeSearchQueryHandlerFactory getInstance() {
@@ -20,7 +22,12 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
 
     @Override
     public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
-        String baseUrl = ServiceList.PeerTube.getBaseUrl();
+        String baseUrl;
+        if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
+            baseUrl = SEPIA_BASE_URL;
+        } else {
+            baseUrl = ServiceList.PeerTube.getBaseUrl();
+        }
         return getUrl(searchString, contentFilters, sortFilter, baseUrl);
     }
 
@@ -38,6 +45,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
 
     @Override
     public String[] getAvailableContentFilter() {
-        return new String[]{VIDEOS};
+        return new String[]{
+                VIDEOS,
+                SEPIA_VIDEOS
+        };
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java

@@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
 
 import javax.annotation.Nullable;
 
@@ -43,6 +44,29 @@ public class PeertubeSearchExtractorTest {
         @Nullable @Override public String expectedSearchSuggestion() { return null; }
     }
 
+    public static class SepiaSearch extends DefaultSearchExtractorTest {
+        private static SearchExtractor extractor;
+        private static final String QUERY = "kde";
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            // setting instance might break test when running in parallel
+            PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
+            extractor = PeerTube.getSearchExtractor(QUERY, singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "");
+            extractor.fetchPage();
+        }
+
+        @Override public SearchExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return PeerTube; }
+        @Override public String expectedName() { return QUERY; }
+        @Override public String expectedId() { return QUERY; }
+        @Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; }
+        @Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return null; }
+    }
+
     public static class PagingTest {
         @Test
         public void duplicatedItemsCheck() throws Exception {

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java

@@ -3,7 +3,9 @@ package org.schabi.newpipe.extractor.services.peertube.search;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
 
+import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
 import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
 
@@ -22,5 +24,7 @@ public class PeertubeSearchQHTest {
         assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
         assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
         assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
+        assertEquals("https://sepiasearch.org/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "").getUrl());
+        assertEquals("https://anotherpeertubeindex.com/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "", "https://anotherpeertubeindex.com").getUrl());
     }
 }