From b6e6f403a80fe4ae691f81bfe53cfc1723de7345 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Thu, 9 Apr 2020 20:37:49 +0300 Subject: [PATCH 1/3] add support for PeerTube channels extraction --- .../services/peertube/PeertubeService.java | 5 +- .../extractors/PeertubeChannelExtractor.java | 4 +- .../PeertubeCommentsInfoItemExtractor.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../PeertubeStreamInfoItemExtractor.java | 3 +- .../extractors/PeertubeUserExtractor.java | 187 ++++++++++++++++ .../PeertubeChannelLinkHandlerFactory.java | 10 +- .../PeertubeChannelExtractorTest.java | 33 ++- ...PeertubeChannelLinkHandlerFactoryTest.java | 4 +- .../peertube/PeertubeUserExtractorTest.java | 205 ++++++++++++++++++ 10 files changed, 425 insertions(+), 30 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 29cc2b1de..53e0f5463 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -77,7 +77,10 @@ public class PeertubeService extends StreamingService { @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return new PeertubeChannelExtractor(this, linkHandler); + + return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ? + new PeertubeUserExtractor(this, linkHandler) : + new PeertubeChannelExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e5acf1ee0..dc27be80c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { - return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id"); } @Override @@ -181,7 +181,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getOriginalUrl() throws ParsingException { - return baseUrl + "/accounts/" + getId(); + return baseUrl + "/" + getId(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index b7cbcec7e..d3b43fae2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -97,7 +97,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getAuthorEndpoint() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 76ead5d61..2b7fa0945 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -128,7 +128,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(json, "account.name"); String host = JsonUtils.getString(json, "account.host"); - return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index c4ed77305..df8b8a609 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -51,7 +51,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java new file mode 100644 index 000000000..2ada58550 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java @@ -0,0 +1,187 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import java.io.IOException; + +public class PeertubeUserExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + private final String baseUrl; + + public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + this.baseUrl = getBaseUrl(); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "avatar.path"); + } catch (Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; + } + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + } catch (Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for (Object c : contents) { + if (c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if (null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if (json != null) { + PeertubeParsingHelper.validate(json); + Number number = JsonUtils.getNumber(json, "total"); + if (number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get PeerTube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if (StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if (nextStart >= total) { + return ""; + } else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + if (null != response && null != response.responseBody()) { + setInitialData(response.responseBody()); + } else { + throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube channel data", e); + } + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return baseUrl + "/" + getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index c1e3f570f..27e0fa593 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -10,8 +10,8 @@ import java.util.List; public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); - private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; - private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)"; + private static final String API_ENDPOINT = "/api/v1/"; public static PeertubeChannelLinkHandlerFactory getInstance() { return instance; @@ -19,7 +19,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - return Parser.matchGroup1(ID_PATTERN, url); + return Parser.matchGroup(ID_PATTERN, url, 0); } @Override @@ -31,11 +31,11 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return baseUrl + ACCOUNTS_ENDPOINT + id; + return baseUrl + API_ENDPOINT + id; } @Override public boolean onAcceptUrl(String url) { - return url.contains("/accounts/"); + return url.contains("/accounts/") || url.contains("/video-channels/"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 461095598..9dc4b013e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; @@ -20,7 +19,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link PeertubeChannelExtractor} */ public class PeertubeChannelExtractorTest { - public static class KDE implements BaseChannelExtractorTest { + public static class DanDAugeTutoriels implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -29,7 +28,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"); extractor.fetchPage(); } @@ -44,22 +43,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("The KDE Community", extractor.getName()); + assertEquals("Dan d'Auge tutoriels", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("kde", extractor.getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -98,16 +97,16 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4); } } - public static class Booteille implements BaseChannelExtractorTest { + public static class Divers implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -116,7 +115,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + .getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457"); extractor.fetchPage(); } @@ -141,22 +140,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("booteille", extractor.getName()); + assertEquals("Divers", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("booteille", extractor.getId()); + assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -195,12 +194,12 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index f9b0c69d1..c44c7548f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -30,7 +30,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void getIdFromUrl() throws ParsingException { - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java new file mode 100644 index 000000000..a1084454a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java @@ -0,0 +1,205 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeUserExtractor; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + +/** + * Test for {@link PeertubeUserExtractor} + */ +public class PeertubeUserExtractorTest { + public static class KDE implements BaseChannelExtractorTest { + private static PeertubeUserExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeUserExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("The KDE Community", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/kde", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class Booteille implements BaseChannelExtractorTest { + private static PeertubeUserExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeUserExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("booteille", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/booteille", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + } + } +} From 2c9f1260eb969fd98d147d5f9684370b855e0764 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Fri, 10 Apr 2020 16:40:53 +0300 Subject: [PATCH 2/3] Fix wb9688 review comments * Rename PeertubeUserExtractor to PeertubeAccountExtractor * Add test for video-channels in PeertubeChannelLinkHandlerFactoryTest * Compatibility support for older versions (use "accounts/" as default) --- .../services/peertube/PeertubeService.java | 8 +++++--- ...xtractor.java => PeertubeAccountExtractor.java} | 4 ++-- .../PeertubeChannelLinkHandlerFactory.java | 8 +++++++- ...Test.java => PeertubeAccountExtractorTest.java} | 14 +++++++------- .../PeertubeChannelLinkHandlerFactoryTest.java | 1 + 5 files changed, 22 insertions(+), 13 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/{PeertubeUserExtractor.java => PeertubeAccountExtractor.java} (97%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/{PeertubeUserExtractorTest.java => PeertubeAccountExtractorTest.java} (95%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 53e0f5463..e025c2be0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -78,9 +78,11 @@ public class PeertubeService extends StreamingService { public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ? - new PeertubeUserExtractor(this, linkHandler) : - new PeertubeChannelExtractor(this, linkHandler); + if (linkHandler.getUrl().contains("/video-channels/")) { + return new PeertubeChannelExtractor(this, linkHandler); + } else { + return new PeertubeAccountExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 2ada58550..ecde0d43c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -21,7 +21,7 @@ import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeUserExtractor extends ChannelExtractor { +public class PeertubeAccountExtractor extends ChannelExtractor { private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; @@ -34,7 +34,7 @@ public class PeertubeUserExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; - public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 27e0fa593..ef81ca4b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -31,7 +31,13 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return baseUrl + API_ENDPOINT + id; + + if (id.matches(ID_PATTERN)) { + return baseUrl + API_ENDPOINT + id; + } else { + // This is needed for compatibility with older versions were we didn't support video channels yet + return baseUrl + API_ENDPOINT + "accounts/" + id; + } } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java similarity index 95% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index a1084454a..fd944f49d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeUserExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -16,18 +16,18 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; /** - * Test for {@link PeertubeUserExtractor} + * Test for {@link PeertubeAccountExtractor} */ -public class PeertubeUserExtractorTest { +public class PeertubeAccountExtractorTest { public static class KDE implements BaseChannelExtractorTest { - private static PeertubeUserExtractor extractor; + private static PeertubeAccountExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeUserExtractor) PeerTube + extractor = (PeertubeAccountExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); extractor.fetchPage(); } @@ -107,14 +107,14 @@ public class PeertubeUserExtractorTest { } public static class Booteille implements BaseChannelExtractorTest { - private static PeertubeUserExtractor extractor; + private static PeertubeAccountExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeUserExtractor) PeerTube + extractor = (PeertubeAccountExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index c44c7548f..28fe1981b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -32,5 +32,6 @@ public class PeertubeChannelLinkHandlerFactoryTest { public void getIdFromUrl() throws ParsingException { assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); } } From 4afe657f6ff97860460dec391adabc0bc62afbb6 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Fri, 10 Apr 2020 19:05:38 +0300 Subject: [PATCH 3/3] Add tests for getUrlFromId and test for video-channels in acceptUrlTest --- .../PeertubeChannelLinkHandlerFactoryTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index 28fe1981b..d47dc6f64 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChanne import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; /** * Test for {@link PeertubeChannelLinkHandlerFactory} @@ -19,6 +20,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @BeforeClass public static void setUp() { + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); NewPipe.init(DownloaderTestImpl.getInstance()); } @@ -26,6 +28,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa")); } @Test @@ -34,4 +37,11 @@ public class PeertubeChannelLinkHandlerFactoryTest { assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); } + + @Test + public void getUrlFromId() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl()); + } }