From 9dfcb3be068773ab3a97ff1580f687f7760b795a Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Thu, 22 Feb 2018 11:52:38 -0300 Subject: [PATCH] Implement SubscriptionExtractor for services - YouTube supports extracting from an export (from subscriptions manager) - SoundCloud supports extracting the "followings" from an user --- NewPipeExtractor.iml | 36 ----- .../newpipe/extractor/StreamingService.java | 2 + .../SoundcloudChannelInfoItemExtractor.java | 18 +-- .../soundcloud/SoundcloudParsingHelper.java | 57 +++++++- .../soundcloud/SoundcloudService.java | 7 + .../SoundcloudSubscriptionExtractor.java | 74 ++++++++++ .../services/youtube/YoutubeService.java | 10 +- .../youtube/YoutubeSubscriptionExtractor.java | 131 ++++++++++++++++++ .../subscription/SubscriptionExtractor.java | 78 +++++++++++ .../subscription/SubscriptionItem.java | 32 +++++ .../SoundcloudSubscriptionExtractorTest.java | 75 ++++++++++ .../YoutubeSubscriptionExtractorTest.java | 89 ++++++++++++ src/test/resources/youtube_export_test.xml | 23 +++ 13 files changed, 582 insertions(+), 50 deletions(-) delete mode 100644 NewPipeExtractor.iml create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionItem.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java create mode 100644 src/test/resources/youtube_export_test.xml diff --git a/NewPipeExtractor.iml b/NewPipeExtractor.iml deleted file mode 100644 index c312e1973..000000000 --- a/NewPipeExtractor.iml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 8bd02d4ac..90a7f7de2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.io.IOException; import java.util.Collections; @@ -71,6 +72,7 @@ public abstract class StreamingService { public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract KioskList getKioskList() throws ExtractionException; + public abstract SubscriptionExtractor getSubscriptionExtractor(); public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { return getChannelExtractor(url, null); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index 532c059b8..ef1799f68 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -4,39 +4,39 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private final JsonObject searchResult; + private final JsonObject itemObject; - public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) { - this.searchResult = searchResult; + public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) { + this.itemObject = itemObject; } @Override public String getName() { - return searchResult.getString("username"); + return itemObject.getString("username"); } @Override public String getUrl() { - return searchResult.getString("permalink_url"); + return itemObject.getString("permalink_url"); } @Override public String getThumbnailUrl() { - return searchResult.getString("avatar_url", ""); + return itemObject.getString("avatar_url", ""); } @Override public long getSubscriberCount() { - return searchResult.getNumber("followers_count", 0).longValue(); + return itemObject.getNumber("followers_count", 0).longValue(); } @Override public long getStreamCount() { - return searchResult.getNumber("track_count", 0).longValue(); + return itemObject.getNumber("track_count", 0).longValue(); } @Override public String getDescription() { - return searchResult.getString("description", ""); + return itemObject.getString("description", ""); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 633f82028..40cc51c74 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -9,20 +9,20 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import javax.annotation.Nonnull; import java.io.IOException; import java.net.URLEncoder; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; -import javax.annotation.Nonnull; - public class SoundcloudParsingHelper { private static String clientId; @@ -100,7 +100,7 @@ public class SoundcloudParsingHelper { /** * Fetch the embed player with the url and return the id (like the id from the json api). * - * @return the id resolved + * @return the resolved id */ public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException { @@ -109,6 +109,57 @@ public class SoundcloudParsingHelper { return Parser.matchGroup1(",\"id\":(.*?),", response); } + /** + * Fetch the users from the given api and commit each of them to the collector. + *

+ * This differ from {@link #getUsersFromApi(ChannelInfoItemCollector, String)} in the sense that they will always + * get MIN_ITEMS or more. + * + * @param minItems the method will return only when it have extracted that many items (equal or more) + */ + public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + String nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); + + while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) { + nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextStreamsUrl); + } + + return nextStreamsUrl; + } + + /** + * Fetch the user items from the given api and commit each of them to the collector. + * + * @return the next streams url, empty if don't have + */ + public static String getUsersFromApi(ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + String response = NewPipe.getDownloader().download(apiUrl); + JsonObject responseObject; + try { + responseObject = JsonParser.object().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } + + JsonArray responseCollection = responseObject.getArray("collection"); + for (Object o : responseCollection) { + if (o instanceof JsonObject) { + JsonObject object = (JsonObject) o; + collector.commit(new SoundcloudChannelInfoItemExtractor(object)); + } + } + + String nextStreamsUrl; + try { + nextStreamsUrl = responseObject.getString("next_href"); + if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); + } catch (Exception ignored) { + nextStreamsUrl = ""; + } + + return nextStreamsUrl; + } + /** * Fetch the streams from the given api and commit each of them to the collector. *

diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 4d86233dd..a5ff0712f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.io.IOException; @@ -92,4 +93,10 @@ public class SoundcloudService extends StreamingService { return list; } + + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return new SoundcloudSubscriptionExtractor(this); + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java new file mode 100644 index 000000000..81af0b3df --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -0,0 +1,74 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionItem; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Extract the "followings" from a user in SoundCloud. + */ +public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { + + public SoundcloudSubscriptionExtractor(SoundcloudService service) { + super(service, Collections.singletonList(ContentSource.CHANNEL_URL)); + } + + @Override + public String getRelatedUrl() { + return "https://soundcloud.com/you"; + } + + @Override + public List fromChannelUrl(String channelUrl) throws IOException, ExtractionException { + if (channelUrl == null) throw new InvalidSourceException("channel url is null"); + + String id; + try { + id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl)); + } catch (ExtractionException e) { + throw new InvalidSourceException(e); + } + + String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings" + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=200"; + ChannelInfoItemCollector collector = new ChannelInfoItemCollector(service.getServiceId()); + // ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough + SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl); + + return toSubscriptionItems(collector.getItemList()); + } + + private String getUrlFrom(String channelUrl) { + channelUrl = channelUrl.replace("http://", "https://").trim(); + + if (!channelUrl.startsWith("https://")) { + if (!channelUrl.contains("soundcloud.com/")) { + channelUrl = "https://soundcloud.com/" + channelUrl; + } else { + channelUrl = "https://" + channelUrl; + } + } + + return channelUrl; + } + + /*////////////////////////////////////////////////////////////////////////// + // Utils + //////////////////////////////////////////////////////////////////////////*/ + + private List toSubscriptionItems(List items) { + List result = new ArrayList<>(items.size()); + for (ChannelInfoItem item : items) { + result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName())); + } + return result; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 26f1e7465..51dc0b122 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.io.IOException; @@ -84,8 +85,7 @@ public class YoutubeService extends StreamingService { } @Override - public KioskList getKioskList() - throws ExtractionException { + public KioskList getKioskList() throws ExtractionException { KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: @@ -104,4 +104,10 @@ public class YoutubeService extends StreamingService { return list; } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return new YoutubeSubscriptionExtractor(this); + } + } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java new file mode 100644 index 000000000..32efe16be --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java @@ -0,0 +1,131 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionItem; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM; + +/** + * Extract subscriptions from a YouTube export (OPML format supported) + */ +public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { + + public YoutubeSubscriptionExtractor(YoutubeService service) { + super(service, Collections.singletonList(INPUT_STREAM)); + } + + @Override + public String getRelatedUrl() { + return "https://www.youtube.com/subscription_manager?action_takeout=1"; + } + + @Override + public List fromInputStream(InputStream contentInputStream) throws ExtractionException { + if (contentInputStream == null) throw new InvalidSourceException("input stream is null"); + + return getItemsFromOPML(contentInputStream); + } + + /*////////////////////////////////////////////////////////////////////////// + // OPML implementation + //////////////////////////////////////////////////////////////////////////*/ + + private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)"; + private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/"; + + private List getItemsFromOPML(InputStream contentInputStream) throws ExtractionException { + final List result = new ArrayList<>(); + + final String contentString = readFromInputStream(contentInputStream); + Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser()); + + if (document.select("opml").isEmpty()) { + throw new InvalidSourceException("document does not have OPML tag"); + } + + if (document.select("outline").isEmpty()) { + throw new InvalidSourceException("document does not have at least one outline tag"); + } + + for (Element outline : document.select("outline[type=rss]")) { + String title = outline.attr("title"); + String xmlUrl = outline.attr("abs:xmlUrl"); + + if (title.isEmpty() || xmlUrl.isEmpty()) { + throw new InvalidSourceException("document has invalid entries"); + } + + try { + String id = Parser.matchGroup1(ID_PATTERN, xmlUrl); + result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title)); + } catch (Parser.RegexException e) { + throw new InvalidSourceException("document has invalid entries", e); + } + } + + return result; + } + + /*////////////////////////////////////////////////////////////////////////// + // Utils + //////////////////////////////////////////////////////////////////////////*/ + + /** + * Throws an exception if the string does not have the right tag/string from a valid export. + */ + private void throwIfTagIsNotFound(String content) throws InvalidSourceException { + if (!content.trim().contains(" 128) { + throwIfTagIsNotFound(contentBuilder.toString()); + hasTag = true; + } + } + } catch (InvalidSourceException e) { + throw e; + } catch (Throwable e) { + throw new InvalidSourceException(e); + } finally { + try { + inputStream.close(); + } catch (IOException ignored) { + } + } + + final String fileContent = contentBuilder.toString().trim(); + if (fileContent.isEmpty()) { + throw new InvalidSourceException("Empty input stream"); + } + + if (!hasTag) { + throwIfTagIsNotFound(fileContent); + } + + return fileContent; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java new file mode 100644 index 000000000..ce91b1e69 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java @@ -0,0 +1,78 @@ +package org.schabi.newpipe.extractor.subscription; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; + +public abstract class SubscriptionExtractor { + + /** + * Exception that should be thrown when the input do not contain valid content that the + * extractor can parse (e.g. nonexistent user in case of a url extraction). + */ + public static class InvalidSourceException extends ParsingException { + public InvalidSourceException() { + this(null, null); + } + + public InvalidSourceException(String detailMessage) { + this(detailMessage, null); + } + + public InvalidSourceException(Throwable cause) { + this(null, cause); + } + + public InvalidSourceException(String detailMessage, Throwable cause) { + super(detailMessage == null ? "Not a valid source" : "Not a valid source (" + detailMessage + ")", cause); + } + } + + public enum ContentSource { + CHANNEL_URL, INPUT_STREAM + } + + private final List supportedSources; + protected final StreamingService service; + + public SubscriptionExtractor(StreamingService service, List supportedSources) { + this.service = service; + this.supportedSources = Collections.unmodifiableList(supportedSources); + } + + public List getSupportedSources() { + return supportedSources; + } + + /** + * Returns an url that can help/guide the user to the file (or channel url) to extract the subscriptions. + *

For example, in YouTube, the export subscriptions url is a good choice to return here.

+ */ + @Nullable + public abstract String getRelatedUrl(); + + /** + * Reads and parse a list of {@link SubscriptionItem} from the given channel url. + * + * @throws InvalidSourceException when the channelUrl doesn't exist or is invalid + */ + public List fromChannelUrl(String channelUrl) throws IOException, ExtractionException { + throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from a channel url"); + } + + /** + * Reads and parse a list of {@link SubscriptionItem} from the given InputStream. + * + * @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed + */ + @SuppressWarnings("RedundantThrows") + public List fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException { + throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream"); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionItem.java b/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionItem.java new file mode 100644 index 000000000..2beee7a9d --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionItem.java @@ -0,0 +1,32 @@ +package org.schabi.newpipe.extractor.subscription; + +import java.io.Serializable; + +public class SubscriptionItem implements Serializable { + private final int serviceId; + private final String url, name; + + public SubscriptionItem(int serviceId, String url, String name) { + this.serviceId = serviceId; + this.url = url; + this.name = name; + } + + public int getServiceId() { + return serviceId; + } + + public String getUrl() { + return url; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) + + "[name=" + name + " > " + serviceId + ":" + url + "]"; + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java new file mode 100644 index 000000000..cbe1948e1 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java @@ -0,0 +1,75 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionItem; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test for {@link SoundcloudSubscriptionExtractor} + */ +public class SoundcloudSubscriptionExtractorTest { + private static SoundcloudSubscriptionExtractor subscriptionExtractor; + private static UrlIdHandler urlHandler; + + @BeforeClass + public static void setupClass() { + NewPipe.init(Downloader.getInstance()); + subscriptionExtractor = new SoundcloudSubscriptionExtractor(ServiceList.SoundCloud); + urlHandler = ServiceList.SoundCloud.getChannelUrlIdHandler(); + } + + @Test + public void testFromChannelUrl() throws Exception { + testList(subscriptionExtractor.fromChannelUrl("https://soundcloud.com/monstercat")); + testList(subscriptionExtractor.fromChannelUrl("http://soundcloud.com/monstercat")); + testList(subscriptionExtractor.fromChannelUrl("soundcloud.com/monstercat")); + testList(subscriptionExtractor.fromChannelUrl("monstercat")); + + //Empty followings user + testList(subscriptionExtractor.fromChannelUrl("some-random-user-184047028")); + } + + @Test + public void testInvalidSourceException() { + List invalidList = Arrays.asList( + "httttps://invalid.com/user", + ".com/monstercat", + "ithinkthatthisuserdontexist", + "", + null + ); + + for (String invalidUser : invalidList) { + try { + subscriptionExtractor.fromChannelUrl(invalidUser); + + fail("didn't throw exception"); + } catch (IOException e) { + // Ignore it, could be an unstable network on the CI server + } catch (Exception e) { + boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException; + assertTrue(e.getClass().getSimpleName() + " is not the expected exception", isExpectedException); + } + } + } + + private void testList(List subscriptionItems) { + for (SubscriptionItem item : subscriptionItems) { + assertNotNull(item.getName()); + assertNotNull(item.getUrl()); + assertTrue(urlHandler.acceptUrl(item.getUrl())); + assertFalse(item.getServiceId() == -1); + } + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java new file mode 100644 index 000000000..2ab515fd8 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -0,0 +1,89 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionItem; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test for {@link YoutubeSubscriptionExtractor} + */ +public class YoutubeSubscriptionExtractorTest { + private static YoutubeSubscriptionExtractor subscriptionExtractor; + private static UrlIdHandler urlHandler; + + @BeforeClass + public static void setupClass() { + NewPipe.init(Downloader.getInstance()); + subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube); + urlHandler = ServiceList.YouTube.getChannelUrlIdHandler(); + } + + @Test + public void testFromInputStream() throws Exception { + File testFile = new File("src/test/resources/youtube_export_test.xml"); + List subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile)); + assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8); + + for (SubscriptionItem item : subscriptionItems) { + assertNotNull(item.getName()); + assertNotNull(item.getUrl()); + assertTrue(urlHandler.acceptUrl(item.getUrl())); + assertFalse(item.getServiceId() == -1); + } + } + + @Test + public void testEmptySourceException() throws Exception { + String emptySource = "" + + "" + + ""; + + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8"))); + assertTrue(items.isEmpty()); + } + + @Test + public void testInvalidSourceException() { + List invalidList = Arrays.asList( + "", + "", + "", + "", + "", + "", + null, + "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28", + "gibberish"); + + for (String invalidContent : invalidList) { + try { + if (invalidContent != null) { + byte[] bytes = invalidContent.getBytes("UTF-8"); + subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); + } else { + subscriptionExtractor.fromInputStream(null); + } + + fail("didn't throw exception"); + } catch (Exception e) { + // System.out.println(" -> " + e); + boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException; + assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException); + } + } + } +} diff --git a/src/test/resources/youtube_export_test.xml b/src/test/resources/youtube_export_test.xml new file mode 100644 index 000000000..4092f98aa --- /dev/null +++ b/src/test/resources/youtube_export_test.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + \ No newline at end of file