Implement SubscriptionExtractor for services
- YouTube supports extracting from an export (from subscriptions manager) - SoundCloud supports extracting the "followings" from an user
This commit is contained in:
Mauricio Colli
parent
a99f466c28
commit
9dfcb3be06
|
|
@ -1,36 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module external.linked.project.id="NewPipeExtractor" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="unspecified" type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7">
|
||||
<output url="file://$MODULE_DIR$/out/production/classes" />
|
||||
<output-test url="file://$MODULE_DIR$/out/test/classes" />
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/build" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/out" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.grack:nanojson:1.1" level="project" />
|
||||
<orderEntry type="library" scope="PROVIDED" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
|
||||
<orderEntry type="library" scope="PROVIDED" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
|
||||
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
|
||||
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.grack:nanojson:1.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: com.grack:nanojson:1.1" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: junit:junit:4.12" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
|
|
@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
|
|||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
|
|
@ -71,6 +72,7 @@ public abstract class StreamingService {
|
|||
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
|
||||
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
|
||||
public abstract KioskList getKioskList() throws ExtractionException;
|
||||
public abstract SubscriptionExtractor getSubscriptionExtractor();
|
||||
|
||||
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
|
||||
return getChannelExtractor(url, null);
|
||||
|
|
|
|||
|
|
@ -4,39 +4,39 @@ import com.grack.nanojson.JsonObject;
|
|||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
|
||||
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
|
||||
private final JsonObject searchResult;
|
||||
private final JsonObject itemObject;
|
||||
|
||||
public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) {
|
||||
this.searchResult = searchResult;
|
||||
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
|
||||
this.itemObject = itemObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return searchResult.getString("username");
|
||||
return itemObject.getString("username");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() {
|
||||
return searchResult.getString("permalink_url");
|
||||
return itemObject.getString("permalink_url");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return searchResult.getString("avatar_url", "");
|
||||
return itemObject.getString("avatar_url", "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() {
|
||||
return searchResult.getNumber("followers_count", 0).longValue();
|
||||
return itemObject.getNumber("followers_count", 0).longValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamCount() {
|
||||
return searchResult.getNumber("track_count", 0).longValue();
|
||||
return itemObject.getNumber("track_count", 0).longValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return searchResult.getString("description", "");
|
||||
return itemObject.getString("description", "");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,20 +9,20 @@ import org.jsoup.nodes.Document;
|
|||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class SoundcloudParsingHelper {
|
||||
private static String clientId;
|
||||
|
||||
|
|
@ -100,7 +100,7 @@ public class SoundcloudParsingHelper {
|
|||
/**
|
||||
* Fetch the embed player with the url and return the id (like the id from the json api).
|
||||
*
|
||||
* @return the id resolved
|
||||
* @return the resolved id
|
||||
*/
|
||||
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
||||
|
|
@ -109,6 +109,57 @@ public class SoundcloudParsingHelper {
|
|||
return Parser.matchGroup1(",\"id\":(.*?),", response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the users from the given api and commit each of them to the collector.
|
||||
* <p>
|
||||
* This differ from {@link #getUsersFromApi(ChannelInfoItemCollector, String)} in the sense that they will always
|
||||
* get MIN_ITEMS or more.
|
||||
*
|
||||
* @param minItems the method will return only when it have extracted that many items (equal or more)
|
||||
*/
|
||||
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
String nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
|
||||
|
||||
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextStreamsUrl);
|
||||
}
|
||||
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the user items from the given api and commit each of them to the collector.
|
||||
*
|
||||
* @return the next streams url, empty if don't have
|
||||
*/
|
||||
public static String getUsersFromApi(ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
String response = NewPipe.getDownloader().download(apiUrl);
|
||||
JsonObject responseObject;
|
||||
try {
|
||||
responseObject = JsonParser.object().from(response);
|
||||
} catch (JsonParserException e) {
|
||||
throw new ParsingException("Could not parse json response", e);
|
||||
}
|
||||
|
||||
JsonArray responseCollection = responseObject.getArray("collection");
|
||||
for (Object o : responseCollection) {
|
||||
if (o instanceof JsonObject) {
|
||||
JsonObject object = (JsonObject) o;
|
||||
collector.commit(new SoundcloudChannelInfoItemExtractor(object));
|
||||
}
|
||||
}
|
||||
|
||||
String nextStreamsUrl;
|
||||
try {
|
||||
nextStreamsUrl = responseObject.getString("next_href");
|
||||
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
|
||||
} catch (Exception ignored) {
|
||||
nextStreamsUrl = "";
|
||||
}
|
||||
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the streams from the given api and commit each of them to the collector.
|
||||
* <p>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
|
|||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
|
@ -92,4 +93,10 @@ public class SoundcloudService extends StreamingService {
|
|||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public SubscriptionExtractor getSubscriptionExtractor() {
|
||||
return new SoundcloudSubscriptionExtractor(this);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Extract the "followings" from a user in SoundCloud.
|
||||
*/
|
||||
public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
|
||||
|
||||
public SoundcloudSubscriptionExtractor(SoundcloudService service) {
|
||||
super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRelatedUrl() {
|
||||
return "https://soundcloud.com/you";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
|
||||
if (channelUrl == null) throw new InvalidSourceException("channel url is null");
|
||||
|
||||
String id;
|
||||
try {
|
||||
id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl));
|
||||
} catch (ExtractionException e) {
|
||||
throw new InvalidSourceException(e);
|
||||
}
|
||||
|
||||
String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=200";
|
||||
ChannelInfoItemCollector collector = new ChannelInfoItemCollector(service.getServiceId());
|
||||
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
|
||||
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
|
||||
|
||||
return toSubscriptionItems(collector.getItemList());
|
||||
}
|
||||
|
||||
private String getUrlFrom(String channelUrl) {
|
||||
channelUrl = channelUrl.replace("http://", "https://").trim();
|
||||
|
||||
if (!channelUrl.startsWith("https://")) {
|
||||
if (!channelUrl.contains("soundcloud.com/")) {
|
||||
channelUrl = "https://soundcloud.com/" + channelUrl;
|
||||
} else {
|
||||
channelUrl = "https://" + channelUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return channelUrl;
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Utils
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) {
|
||||
List<SubscriptionItem> result = new ArrayList<>(items.size());
|
||||
for (ChannelInfoItem item : items) {
|
||||
result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
|
|||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
|
@ -84,8 +85,7 @@ public class YoutubeService extends StreamingService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public KioskList getKioskList()
|
||||
throws ExtractionException {
|
||||
public KioskList getKioskList() throws ExtractionException {
|
||||
KioskList list = new KioskList(getServiceId());
|
||||
|
||||
// add kiosks here e.g.:
|
||||
|
|
@ -104,4 +104,10 @@ public class YoutubeService extends StreamingService {
|
|||
|
||||
return list;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SubscriptionExtractor getSubscriptionExtractor() {
|
||||
return new YoutubeSubscriptionExtractor(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM;
|
||||
|
||||
/**
|
||||
* Extract subscriptions from a YouTube export (OPML format supported)
|
||||
*/
|
||||
public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
|
||||
|
||||
public YoutubeSubscriptionExtractor(YoutubeService service) {
|
||||
super(service, Collections.singletonList(INPUT_STREAM));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRelatedUrl() {
|
||||
return "https://www.youtube.com/subscription_manager?action_takeout=1";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws ExtractionException {
|
||||
if (contentInputStream == null) throw new InvalidSourceException("input stream is null");
|
||||
|
||||
return getItemsFromOPML(contentInputStream);
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// OPML implementation
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)";
|
||||
private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
|
||||
|
||||
private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream) throws ExtractionException {
|
||||
final List<SubscriptionItem> result = new ArrayList<>();
|
||||
|
||||
final String contentString = readFromInputStream(contentInputStream);
|
||||
Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser());
|
||||
|
||||
if (document.select("opml").isEmpty()) {
|
||||
throw new InvalidSourceException("document does not have OPML tag");
|
||||
}
|
||||
|
||||
if (document.select("outline").isEmpty()) {
|
||||
throw new InvalidSourceException("document does not have at least one outline tag");
|
||||
}
|
||||
|
||||
for (Element outline : document.select("outline[type=rss]")) {
|
||||
String title = outline.attr("title");
|
||||
String xmlUrl = outline.attr("abs:xmlUrl");
|
||||
|
||||
if (title.isEmpty() || xmlUrl.isEmpty()) {
|
||||
throw new InvalidSourceException("document has invalid entries");
|
||||
}
|
||||
|
||||
try {
|
||||
String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
|
||||
result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
|
||||
} catch (Parser.RegexException e) {
|
||||
throw new InvalidSourceException("document has invalid entries", e);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Utils
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
/**
|
||||
* Throws an exception if the string does not have the right tag/string from a valid export.
|
||||
*/
|
||||
private void throwIfTagIsNotFound(String content) throws InvalidSourceException {
|
||||
if (!content.trim().contains("<opml")) {
|
||||
throw new InvalidSourceException("input stream does not have OPML tag");
|
||||
}
|
||||
}
|
||||
|
||||
private String readFromInputStream(InputStream inputStream) throws InvalidSourceException {
|
||||
StringBuilder contentBuilder = new StringBuilder();
|
||||
boolean hasTag = false;
|
||||
try {
|
||||
byte[] buffer = new byte[16 * 1024];
|
||||
int read;
|
||||
while ((read = inputStream.read(buffer)) != -1) {
|
||||
String currentPartOfContent = new String(buffer, 0, read, "UTF-8");
|
||||
contentBuilder.append(currentPartOfContent);
|
||||
|
||||
// Fail-fast in case of reading a long unsupported input stream
|
||||
if (!hasTag && contentBuilder.length() > 128) {
|
||||
throwIfTagIsNotFound(contentBuilder.toString());
|
||||
hasTag = true;
|
||||
}
|
||||
}
|
||||
} catch (InvalidSourceException e) {
|
||||
throw e;
|
||||
} catch (Throwable e) {
|
||||
throw new InvalidSourceException(e);
|
||||
} finally {
|
||||
try {
|
||||
inputStream.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
final String fileContent = contentBuilder.toString().trim();
|
||||
if (fileContent.isEmpty()) {
|
||||
throw new InvalidSourceException("Empty input stream");
|
||||
}
|
||||
|
||||
if (!hasTag) {
|
||||
throwIfTagIsNotFound(fileContent);
|
||||
}
|
||||
|
||||
return fileContent;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package org.schabi.newpipe.extractor.subscription;
|
||||
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class SubscriptionExtractor {
|
||||
|
||||
/**
|
||||
* Exception that should be thrown when the input <b>do not</b> contain valid content that the
|
||||
* extractor can parse (e.g. nonexistent user in case of a url extraction).
|
||||
*/
|
||||
public static class InvalidSourceException extends ParsingException {
|
||||
public InvalidSourceException() {
|
||||
this(null, null);
|
||||
}
|
||||
|
||||
public InvalidSourceException(String detailMessage) {
|
||||
this(detailMessage, null);
|
||||
}
|
||||
|
||||
public InvalidSourceException(Throwable cause) {
|
||||
this(null, cause);
|
||||
}
|
||||
|
||||
public InvalidSourceException(String detailMessage, Throwable cause) {
|
||||
super(detailMessage == null ? "Not a valid source" : "Not a valid source (" + detailMessage + ")", cause);
|
||||
}
|
||||
}
|
||||
|
||||
public enum ContentSource {
|
||||
CHANNEL_URL, INPUT_STREAM
|
||||
}
|
||||
|
||||
private final List<ContentSource> supportedSources;
|
||||
protected final StreamingService service;
|
||||
|
||||
public SubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {
|
||||
this.service = service;
|
||||
this.supportedSources = Collections.unmodifiableList(supportedSources);
|
||||
}
|
||||
|
||||
public List<ContentSource> getSupportedSources() {
|
||||
return supportedSources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an url that can help/guide the user to the file (or channel url) to extract the subscriptions.
|
||||
* <p>For example, in YouTube, the export subscriptions url is a good choice to return here.</p>
|
||||
*/
|
||||
@Nullable
|
||||
public abstract String getRelatedUrl();
|
||||
|
||||
/**
|
||||
* Reads and parse a list of {@link SubscriptionItem} from the given channel url.
|
||||
*
|
||||
* @throws InvalidSourceException when the channelUrl doesn't exist or is invalid
|
||||
*/
|
||||
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
|
||||
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from a channel url");
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads and parse a list of {@link SubscriptionItem} from the given InputStream.
|
||||
*
|
||||
* @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed
|
||||
*/
|
||||
@SuppressWarnings("RedundantThrows")
|
||||
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException {
|
||||
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package org.schabi.newpipe.extractor.subscription;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class SubscriptionItem implements Serializable {
|
||||
private final int serviceId;
|
||||
private final String url, name;
|
||||
|
||||
public SubscriptionItem(int serviceId, String url, String name) {
|
||||
this.serviceId = serviceId;
|
||||
this.url = url;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public int getServiceId() {
|
||||
return serviceId;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) +
|
||||
"[name=" + name + " > " + serviceId + ":" + url + "]";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
* Test for {@link SoundcloudSubscriptionExtractor}
|
||||
*/
|
||||
public class SoundcloudSubscriptionExtractorTest {
|
||||
private static SoundcloudSubscriptionExtractor subscriptionExtractor;
|
||||
private static UrlIdHandler urlHandler;
|
||||
|
||||
@BeforeClass
|
||||
public static void setupClass() {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
subscriptionExtractor = new SoundcloudSubscriptionExtractor(ServiceList.SoundCloud);
|
||||
urlHandler = ServiceList.SoundCloud.getChannelUrlIdHandler();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFromChannelUrl() throws Exception {
|
||||
testList(subscriptionExtractor.fromChannelUrl("https://soundcloud.com/monstercat"));
|
||||
testList(subscriptionExtractor.fromChannelUrl("http://soundcloud.com/monstercat"));
|
||||
testList(subscriptionExtractor.fromChannelUrl("soundcloud.com/monstercat"));
|
||||
testList(subscriptionExtractor.fromChannelUrl("monstercat"));
|
||||
|
||||
//Empty followings user
|
||||
testList(subscriptionExtractor.fromChannelUrl("some-random-user-184047028"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidSourceException() {
|
||||
List<String> invalidList = Arrays.asList(
|
||||
"httttps://invalid.com/user",
|
||||
".com/monstercat",
|
||||
"ithinkthatthisuserdontexist",
|
||||
"",
|
||||
null
|
||||
);
|
||||
|
||||
for (String invalidUser : invalidList) {
|
||||
try {
|
||||
subscriptionExtractor.fromChannelUrl(invalidUser);
|
||||
|
||||
fail("didn't throw exception");
|
||||
} catch (IOException e) {
|
||||
// Ignore it, could be an unstable network on the CI server
|
||||
} catch (Exception e) {
|
||||
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
|
||||
assertTrue(e.getClass().getSimpleName() + " is not the expected exception", isExpectedException);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void testList(List<SubscriptionItem> subscriptionItems) {
|
||||
for (SubscriptionItem item : subscriptionItems) {
|
||||
assertNotNull(item.getName());
|
||||
assertNotNull(item.getUrl());
|
||||
assertTrue(urlHandler.acceptUrl(item.getUrl()));
|
||||
assertFalse(item.getServiceId() == -1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeSubscriptionExtractor}
|
||||
*/
|
||||
public class YoutubeSubscriptionExtractorTest {
|
||||
private static YoutubeSubscriptionExtractor subscriptionExtractor;
|
||||
private static UrlIdHandler urlHandler;
|
||||
|
||||
@BeforeClass
|
||||
public static void setupClass() {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
|
||||
urlHandler = ServiceList.YouTube.getChannelUrlIdHandler();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFromInputStream() throws Exception {
|
||||
File testFile = new File("src/test/resources/youtube_export_test.xml");
|
||||
List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile));
|
||||
assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8);
|
||||
|
||||
for (SubscriptionItem item : subscriptionItems) {
|
||||
assertNotNull(item.getName());
|
||||
assertNotNull(item.getUrl());
|
||||
assertTrue(urlHandler.acceptUrl(item.getUrl()));
|
||||
assertFalse(item.getServiceId() == -1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptySourceException() throws Exception {
|
||||
String emptySource = "<opml version=\"1.1\"><body>" +
|
||||
"<outline text=\"Testing\" title=\"123\" />" +
|
||||
"</body></opml>";
|
||||
|
||||
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8")));
|
||||
assertTrue(items.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidSourceException() {
|
||||
List<String> invalidList = Arrays.asList(
|
||||
"<xml><notvalid></notvalid></xml>",
|
||||
"<opml><notvalid></notvalid></opml>",
|
||||
"<opml><body></body></opml>",
|
||||
"<opml><body><outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/></outline></body></opml>",
|
||||
"<opml><body><outline><outline text=\"invalid\" title=\"url\" type=\"rss\"" +
|
||||
" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/></outline></body></opml>",
|
||||
"",
|
||||
null,
|
||||
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
|
||||
"gibberish");
|
||||
|
||||
for (String invalidContent : invalidList) {
|
||||
try {
|
||||
if (invalidContent != null) {
|
||||
byte[] bytes = invalidContent.getBytes("UTF-8");
|
||||
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
|
||||
} else {
|
||||
subscriptionExtractor.fromInputStream(null);
|
||||
}
|
||||
|
||||
fail("didn't throw exception");
|
||||
} catch (Exception e) {
|
||||
// System.out.println(" -> " + e);
|
||||
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
|
||||
assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<opml version="1.1">
|
||||
<body>
|
||||
<outline text="YouTube Subscriptions" title="YouTube Subscriptions">
|
||||
<outline text="Kurzgesagt – In a Nutshell" title="Kurzgesagt – In a Nutshell" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
|
||||
<outline text="CaptainDisillusion" title="CaptainDisillusion" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg"/>
|
||||
<outline text="TED" title="TED" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCAuUUnT6oDeKwE6v1NGQxug"/>
|
||||
<outline text="Gorillaz" title="Gorillaz" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCfIXdjDQH9Fau7y99_Orpjw"/>
|
||||
<outline text="ElectroBOOM" title="ElectroBOOM" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCJ0-OtVpF0wOKEqT2Z1HEtA"/>
|
||||
|
||||
<outline text="ⓤⓝⓘⓒⓞⓓⓔ" title="ⓤⓝⓘⓒⓞⓓⓔ" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
|
||||
<outline text="中文" title="中文" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
|
||||
<outline text="हिंदी" title="हिंदी" type="rss"
|
||||
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
|
||||
</outline>
|
||||
</body>
|
||||
</opml>
|
||||
Loading…
Reference in New Issue