Refactor and improvements
This commit is contained in:
Mauricio Colli
parent
731102be39
commit
ea5b08db4c
|
|
@ -10,5 +10,8 @@ dependencies {
|
|||
implementation 'org.mozilla:rhino:1.7.7.1'
|
||||
|
||||
testImplementation 'junit:junit:4.12'
|
||||
|
||||
sourceCompatibility = 1.7
|
||||
targetCompatibility = 1.7
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,35 +1,73 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.io.IOException;
|
||||
|
||||
public abstract class Extractor implements Serializable {
|
||||
private final int serviceId;
|
||||
private final String url;
|
||||
private final UrlIdHandler urlIdHandler;
|
||||
private final StreamInfoItemCollector previewInfoCollector;
|
||||
public abstract class Extractor {
|
||||
/**
|
||||
* {@link StreamingService} currently related to this extractor.<br/>
|
||||
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
|
||||
*/
|
||||
private final StreamingService service;
|
||||
|
||||
public Extractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
|
||||
this.urlIdHandler = urlIdHandler;
|
||||
this.serviceId = serviceId;
|
||||
this.url = url;
|
||||
this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
|
||||
/**
|
||||
* Dirty/original url that was passed in the constructor.
|
||||
* <p>
|
||||
* What makes a url "dirty" or not is, for example, the additional parameters
|
||||
* (not important as—in this case—the id):
|
||||
* <pre>
|
||||
* https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i> → <i><b>&t=4s</b></i>
|
||||
* </pre>
|
||||
* But as you can imagine, the time parameter is very important when calling, for example, {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
|
||||
*/
|
||||
private final String originalUrl;
|
||||
|
||||
/**
|
||||
* The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
|
||||
* <p>
|
||||
* Is lazily-cleaned by calling {@link #getCleanUrl()}
|
||||
*/
|
||||
private String cleanUrl;
|
||||
|
||||
public Extractor(StreamingService service, String url) throws ExtractionException {
|
||||
this.service = service;
|
||||
this.originalUrl = url;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
/**
|
||||
* @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
|
||||
*/
|
||||
protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
|
||||
|
||||
/**
|
||||
* Fetch the current page.
|
||||
*/
|
||||
public abstract void fetchPage() throws IOException, ExtractionException;
|
||||
|
||||
public String getOriginalUrl() {
|
||||
return originalUrl;
|
||||
}
|
||||
|
||||
public UrlIdHandler getUrlIdHandler() {
|
||||
return urlIdHandler;
|
||||
public String getCleanUrl() {
|
||||
if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
|
||||
|
||||
try {
|
||||
cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
|
||||
} catch (Exception e) {
|
||||
cleanUrl = null;
|
||||
// Fallback to the original url
|
||||
return originalUrl;
|
||||
}
|
||||
return cleanUrl;
|
||||
}
|
||||
|
||||
public StreamingService getService() {
|
||||
return service;
|
||||
}
|
||||
|
||||
public int getServiceId() {
|
||||
return serviceId;
|
||||
}
|
||||
|
||||
protected StreamInfoItemCollector getStreamPreviewInfoCollector() {
|
||||
return previewInfoCollector;
|
||||
return service.getServiceId();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
public abstract class Info implements Serializable {
|
||||
|
||||
|
|
@ -15,5 +15,5 @@ public abstract class Info implements Serializable {
|
|||
public String url;
|
||||
public String name;
|
||||
|
||||
public List<Throwable> errors = new Vector<>();
|
||||
public List<Throwable> errors = new ArrayList<>();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor;
|
|||
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.02.17.
|
||||
|
|
@ -26,8 +26,8 @@ import java.util.Vector;
|
|||
*/
|
||||
|
||||
public abstract class InfoItemCollector {
|
||||
private List<InfoItem> itemList = new Vector<>();
|
||||
private List<Throwable> errors = new Vector<>();
|
||||
private List<InfoItem> itemList = new ArrayList<>();
|
||||
private List<Throwable> errors = new ArrayList<>();
|
||||
private int serviceId = -1;
|
||||
|
||||
public InfoItemCollector(int serviceId) {
|
||||
|
|
@ -46,7 +46,7 @@ public abstract class InfoItemCollector {
|
|||
if (serviceId != otherC.serviceId) {
|
||||
throw new ExtractionException("Service Id does not equal: "
|
||||
+ NewPipe.getNameOfService(serviceId)
|
||||
+ " and " + NewPipe.getNameOfService(otherC.serviceId));
|
||||
+ " and " + NewPipe.getNameOfService((otherC.serviceId)));
|
||||
}
|
||||
errors.addAll(otherC.errors);
|
||||
itemList.addAll(otherC.itemList);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Base class to extractors that have a list (e.g. playlists, channels).
|
||||
|
|
@ -11,16 +12,40 @@ import java.io.IOException;
|
|||
public abstract class ListExtractor extends Extractor {
|
||||
protected String nextStreamsUrl;
|
||||
|
||||
public ListExtractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
|
||||
super(urlIdHandler, serviceId, url);
|
||||
/**
|
||||
* Get a new ListExtractor with the given nextStreamsUrl set.
|
||||
* <p>
|
||||
* The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
|
||||
* <p>
|
||||
* You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
|
||||
* downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
|
||||
*/
|
||||
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url);
|
||||
setNextStreamsUrl(nextStreamsUrl);
|
||||
|
||||
if (fetchPageUponCreation()) {
|
||||
fetchPage();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean hasMoreStreams(){
|
||||
/**
|
||||
* Decide if the page will be fetched upon creation.
|
||||
* <p>
|
||||
* The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
|
||||
* don't need or know what is the next page, thus, fetch the page).
|
||||
*/
|
||||
protected boolean fetchPageUponCreation() {
|
||||
return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
|
||||
}
|
||||
|
||||
public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException;
|
||||
public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException;
|
||||
|
||||
public boolean hasMoreStreams() {
|
||||
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
|
||||
}
|
||||
|
||||
public abstract StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException;
|
||||
|
||||
public String getNextStreamsUrl() {
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
|
|
@ -29,4 +54,29 @@ public abstract class ListExtractor extends Extractor {
|
|||
this.nextStreamsUrl = nextStreamsUrl;
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Inner
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public static class NextItemsResult {
|
||||
/**
|
||||
* The current list of items to this result
|
||||
*/
|
||||
public final List<InfoItem> nextItemsList;
|
||||
|
||||
/**
|
||||
* Next url to fetch more items
|
||||
*/
|
||||
public final String nextItemsUrl;
|
||||
|
||||
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
|
||||
this.nextItemsList = nextItemsList;
|
||||
this.nextItemsUrl = nextItemsUrl;
|
||||
}
|
||||
|
||||
public boolean hasMoreStreams() {
|
||||
return nextItemsUrl != null && !nextItemsUrl.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public abstract class ListInfo extends Info {
|
||||
public List<InfoItem> related_streams;
|
||||
public boolean has_more_streams;
|
||||
public String next_streams_url;
|
||||
}
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 23.08.15.
|
||||
*
|
||||
|
|
@ -22,54 +20,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Provides access to the video streaming services supported by NewPipe.
|
||||
* Currently only Youtube until the API becomes more stable.
|
||||
*/
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
@SuppressWarnings("ALL")
|
||||
/**
|
||||
* Provides access to streaming services supported by NewPipe.
|
||||
*/
|
||||
public class NewPipe {
|
||||
private static final String TAG = NewPipe.class.toString();
|
||||
|
||||
private NewPipe() {
|
||||
}
|
||||
|
||||
private static Downloader downloader = null;
|
||||
|
||||
public static StreamingService[] getServices() {
|
||||
return ServiceList.serviceList;
|
||||
}
|
||||
|
||||
public static StreamingService getService(int serviceId) throws ExtractionException {
|
||||
for (StreamingService s : ServiceList.serviceList) {
|
||||
if (s.getServiceId() == serviceId) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static StreamingService getService(String serviceName) throws ExtractionException {
|
||||
return ServiceList.serviceList[getIdOfService(serviceName)];
|
||||
}
|
||||
|
||||
public static String getNameOfService(int id) {
|
||||
try {
|
||||
return getService(id).getServiceInfo().name;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Service id not known");
|
||||
e.printStackTrace();
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
public static int getIdOfService(String serviceName) {
|
||||
for (int i = 0; i < ServiceList.serviceList.length; i++) {
|
||||
if (ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
private NewPipe() {
|
||||
}
|
||||
|
||||
public static void init(Downloader d) {
|
||||
|
|
@ -80,12 +40,63 @@ public class NewPipe {
|
|||
return downloader;
|
||||
}
|
||||
|
||||
public static StreamingService getServiceByUrl(String url) {
|
||||
for (StreamingService s : ServiceList.serviceList) {
|
||||
if (s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
|
||||
return s;
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Utils
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public static StreamingService[] getServices() {
|
||||
final ServiceList[] values = ServiceList.values();
|
||||
final StreamingService[] streamingServices = new StreamingService[values.length];
|
||||
|
||||
for (int i = 0; i < values.length; i++) streamingServices[i] = values[i].getService();
|
||||
|
||||
return streamingServices;
|
||||
}
|
||||
|
||||
public static StreamingService getService(int serviceId) throws ExtractionException {
|
||||
for (ServiceList item : ServiceList.values()) {
|
||||
if (item.getService().getServiceId() == serviceId) {
|
||||
return item.getService();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
throw new ExtractionException("There's no service with the id = \"" + serviceId + "\"");
|
||||
}
|
||||
|
||||
public static StreamingService getService(String serviceName) throws ExtractionException {
|
||||
for (ServiceList item : ServiceList.values()) {
|
||||
if (item.getService().getServiceInfo().name.equals(serviceName)) {
|
||||
return item.getService();
|
||||
}
|
||||
}
|
||||
throw new ExtractionException("There's no service with the name = \"" + serviceName + "\"");
|
||||
}
|
||||
|
||||
public static StreamingService getServiceByUrl(String url) throws ExtractionException {
|
||||
for (ServiceList item : ServiceList.values()) {
|
||||
if (item.getService().getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
|
||||
return item.getService();
|
||||
}
|
||||
}
|
||||
throw new ExtractionException("No service can handle the url = \"" + url + "\"");
|
||||
}
|
||||
|
||||
public static int getIdOfService(String serviceName) {
|
||||
try {
|
||||
//noinspection ConstantConditions
|
||||
return getService(serviceName).getServiceId();
|
||||
} catch (ExtractionException ignored) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
public static String getNameOfService(int id) {
|
||||
try {
|
||||
//noinspection ConstantConditions
|
||||
return getService(id).getServiceInfo().name;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Service id not known");
|
||||
e.printStackTrace();
|
||||
return "<unknown>";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,36 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
|
||||
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
|
||||
|
||||
/*
|
||||
* Created by the-scrabi on 18.02.17.
|
||||
/**
|
||||
* A list of supported services.
|
||||
*/
|
||||
public enum ServiceList {
|
||||
Youtube(new YoutubeService(0, "Youtube")),
|
||||
SoundCloud(new SoundcloudService(1, "SoundCloud"));
|
||||
// DailyMotion(new DailyMotionService(2, "DailyMotion"));
|
||||
|
||||
class ServiceList {
|
||||
public static final StreamingService[] serviceList = {
|
||||
new YoutubeService(0),
|
||||
new SoundcloudService(1)
|
||||
};
|
||||
private final StreamingService service;
|
||||
|
||||
ServiceList(StreamingService service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public StreamingService getService() {
|
||||
return service;
|
||||
}
|
||||
|
||||
public StreamingService.ServiceInfo getServiceInfo() {
|
||||
return service.getServiceInfo();
|
||||
}
|
||||
|
||||
public int getId() {
|
||||
return service.getServiceId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return service.getServiceInfo().name;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,11 @@ import java.io.IOException;
|
|||
|
||||
public abstract class StreamingService {
|
||||
public class ServiceInfo {
|
||||
public String name = "";
|
||||
public final String name;
|
||||
|
||||
public ServiceInfo(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
||||
public enum LinkType {
|
||||
|
|
@ -20,35 +24,46 @@ public abstract class StreamingService {
|
|||
PLAYLIST
|
||||
}
|
||||
|
||||
private int serviceId;
|
||||
private final int serviceId;
|
||||
private final ServiceInfo serviceInfo;
|
||||
|
||||
public StreamingService(int id) {
|
||||
serviceId = id;
|
||||
public StreamingService(int id, String name) {
|
||||
this.serviceId = id;
|
||||
this.serviceInfo = new ServiceInfo(name);
|
||||
}
|
||||
|
||||
public abstract ServiceInfo getServiceInfo();
|
||||
|
||||
public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
|
||||
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
|
||||
public abstract UrlIdHandler getPlaylistUrlIdHandlerInstance();
|
||||
public abstract SearchEngine getSearchEngineInstance();
|
||||
public abstract SuggestionExtractor getSuggestionExtractorInstance();
|
||||
public abstract StreamExtractor getStreamExtractorInstance(String url) throws IOException, ExtractionException;
|
||||
public abstract ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException;
|
||||
public abstract PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException;
|
||||
|
||||
|
||||
public final int getServiceId() {
|
||||
return serviceId;
|
||||
}
|
||||
|
||||
public ServiceInfo getServiceInfo() {
|
||||
return serviceInfo;
|
||||
}
|
||||
|
||||
public abstract UrlIdHandler getStreamUrlIdHandler();
|
||||
public abstract UrlIdHandler getChannelUrlIdHandler();
|
||||
public abstract UrlIdHandler getPlaylistUrlIdHandler();
|
||||
public abstract SearchEngine getSearchEngine();
|
||||
public abstract SuggestionExtractor getSuggestionExtractor();
|
||||
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
|
||||
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
|
||||
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
|
||||
|
||||
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
|
||||
return getChannelExtractor(url, null);
|
||||
}
|
||||
|
||||
public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
|
||||
return getPlaylistExtractor(url, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* figure out where the link is pointing to (a channel, video, playlist, etc.)
|
||||
*/
|
||||
public final LinkType getLinkTypeByUrl(String url) {
|
||||
UrlIdHandler sH = getStreamUrlIdHandlerInstance();
|
||||
UrlIdHandler cH = getChannelUrlIdHandlerInstance();
|
||||
UrlIdHandler pH = getPlaylistUrlIdHandlerInstance();
|
||||
UrlIdHandler sH = getStreamUrlIdHandler();
|
||||
UrlIdHandler cH = getChannelUrlIdHandler();
|
||||
UrlIdHandler pH = getPlaylistUrlIdHandler();
|
||||
|
||||
if (sH.acceptUrl(url)) {
|
||||
return LinkType.STREAM;
|
||||
|
|
|
|||
|
|
@ -33,8 +33,7 @@ public abstract class SuggestionExtractor {
|
|||
this.serviceId = serviceId;
|
||||
}
|
||||
|
||||
public abstract List<String> suggestionList(String query, String contentCountry)
|
||||
throws ExtractionException, IOException;
|
||||
public abstract List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException;
|
||||
|
||||
public int getServiceId() {
|
||||
return serviceId;
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 26.07.16.
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
package org.schabi.newpipe.extractor.channel;
|
||||
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
|
@ -31,8 +30,13 @@ import java.io.IOException;
|
|||
|
||||
public abstract class ChannelExtractor extends ListExtractor {
|
||||
|
||||
public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, serviceId, url);
|
||||
public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
|
||||
return getService().getChannelUrlIdHandler();
|
||||
}
|
||||
|
||||
public abstract String getChannelId() throws ParsingException;
|
||||
|
|
@ -40,7 +44,6 @@ public abstract class ChannelExtractor extends ListExtractor {
|
|||
public abstract String getAvatarUrl() throws ParsingException;
|
||||
public abstract String getBannerUrl() throws ParsingException;
|
||||
public abstract String getFeedUrl() throws ParsingException;
|
||||
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
|
||||
public abstract long getSubscriberCount() throws ParsingException;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
package org.schabi.newpipe.extractor.channel;
|
||||
|
||||
import org.schabi.newpipe.extractor.Info;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
|
||||
import org.schabi.newpipe.extractor.ListInfo;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.util.List;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 31.07.16.
|
||||
|
|
@ -27,17 +32,36 @@ import java.util.List;
|
|||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class ChannelInfo extends Info {
|
||||
public class ChannelInfo extends ListInfo {
|
||||
|
||||
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
|
||||
}
|
||||
|
||||
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return service.getChannelExtractor(null, nextStreamsUrl).getNextStreams();
|
||||
}
|
||||
|
||||
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
|
||||
return getInfo(NewPipe.getServiceByUrl(url), url);
|
||||
}
|
||||
|
||||
public static ChannelInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
|
||||
return getInfo(serviceItem.getService(), url);
|
||||
}
|
||||
|
||||
public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
return getInfo(service.getChannelExtractor(url));
|
||||
}
|
||||
|
||||
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
|
||||
ChannelInfo info = new ChannelInfo();
|
||||
|
||||
// important data
|
||||
info.service_id = extractor.getServiceId();
|
||||
info.url = extractor.getUrl();
|
||||
info.url = extractor.getCleanUrl();
|
||||
info.id = extractor.getChannelId();
|
||||
info.name = extractor.getChannelName();
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
|
||||
try {
|
||||
info.avatar_url = extractor.getAvatarUrl();
|
||||
|
|
@ -67,13 +91,16 @@ public class ChannelInfo extends Info {
|
|||
info.errors.add(e);
|
||||
}
|
||||
|
||||
// Lists can be null if a exception was thrown during extraction
|
||||
if (info.related_streams == null) info.related_streams = new ArrayList<>();
|
||||
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
info.next_streams_url = extractor.getNextStreamsUrl();
|
||||
return info;
|
||||
}
|
||||
|
||||
public String avatar_url;
|
||||
public String banner_url;
|
||||
public String feed_url;
|
||||
public List<InfoItem> related_streams;
|
||||
public long subscriber_count = -1;
|
||||
public boolean has_more_streams = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem {
|
|||
public String thumbnail_url;
|
||||
public String description;
|
||||
public long subscriber_count = -1;
|
||||
public long view_count = -1;
|
||||
public long stream_count = -1;
|
||||
|
||||
public ChannelInfoItem() {
|
||||
super(InfoType.CHANNEL);
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
|
|||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.view_count = extractor.getViewCount();
|
||||
resultItem.stream_count = extractor.getStreamCount();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,5 +28,5 @@ public interface ChannelInfoItemExtractor {
|
|||
String getWebPageUrl() throws ParsingException;
|
||||
String getDescription() throws ParsingException;
|
||||
long getSubscriberCount() throws ParsingException;
|
||||
long getViewCount() throws ParsingException;
|
||||
long getStreamCount() throws ParsingException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,22 @@
|
|||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public abstract class PlaylistExtractor extends ListExtractor {
|
||||
|
||||
public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, serviceId, url);
|
||||
public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
|
||||
return getService().getPlaylistUrlIdHandler();
|
||||
}
|
||||
|
||||
public abstract String getPlaylistId() throws ParsingException;
|
||||
|
|
@ -22,6 +26,5 @@ public abstract class PlaylistExtractor extends ListExtractor {
|
|||
public abstract String getUploaderUrl() throws ParsingException;
|
||||
public abstract String getUploaderName() throws ParsingException;
|
||||
public abstract String getUploaderAvatarUrl() throws ParsingException;
|
||||
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
|
||||
public abstract long getStreamsCount() throws ParsingException;
|
||||
public abstract long getStreamCount() throws ParsingException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,49 @@
|
|||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.Info;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
|
||||
import org.schabi.newpipe.extractor.ListInfo;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.util.List;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
public class PlaylistInfo extends Info {
|
||||
public class PlaylistInfo extends ListInfo {
|
||||
|
||||
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
|
||||
}
|
||||
|
||||
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return service.getPlaylistExtractor(null, nextStreamsUrl).getNextStreams();
|
||||
}
|
||||
|
||||
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
|
||||
return getInfo(NewPipe.getServiceByUrl(url), url);
|
||||
}
|
||||
|
||||
public static PlaylistInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
|
||||
return getInfo(serviceItem.getService(), url);
|
||||
}
|
||||
|
||||
public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
return getInfo(service.getPlaylistExtractor(url));
|
||||
}
|
||||
|
||||
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
|
||||
PlaylistInfo info = new PlaylistInfo();
|
||||
|
||||
info.service_id = extractor.getServiceId();
|
||||
info.url = extractor.getUrl();
|
||||
info.url = extractor.getCleanUrl();
|
||||
info.id = extractor.getPlaylistId();
|
||||
info.name = extractor.getPlaylistName();
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
|
||||
try {
|
||||
info.streams_count = extractor.getStreamsCount();
|
||||
info.stream_count = extractor.getStreamCount();
|
||||
} catch (Exception e) {
|
||||
info.errors.add(e);
|
||||
}
|
||||
|
|
@ -56,6 +80,11 @@ public class PlaylistInfo extends Info {
|
|||
info.errors.add(e);
|
||||
}
|
||||
|
||||
// Lists can be null if a exception was thrown during extraction
|
||||
if (info.related_streams == null) info.related_streams = new ArrayList<>();
|
||||
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
info.next_streams_url = extractor.getNextStreamsUrl();
|
||||
return info;
|
||||
}
|
||||
|
||||
|
|
@ -64,7 +93,5 @@ public class PlaylistInfo extends Info {
|
|||
public String uploader_url;
|
||||
public String uploader_name;
|
||||
public String uploader_avatar_url;
|
||||
public long streams_count = 0;
|
||||
public List<InfoItem> related_streams;
|
||||
public boolean has_more_streams;
|
||||
public long stream_count = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ public class PlaylistInfoItem extends InfoItem {
|
|||
/**
|
||||
* How many streams this playlist have
|
||||
*/
|
||||
public long streams_count = 0;
|
||||
public long stream_count = 0;
|
||||
|
||||
public PlaylistInfoItem() {
|
||||
super(InfoType.PLAYLIST);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
public class PlaylistInfoItemCollector extends InfoItemCollector {
|
||||
|
|
@ -22,7 +21,7 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
|
|||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.streams_count = extractor.getStreamsCount();
|
||||
resultItem.stream_count = extractor.getStreamCount();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@ public interface PlaylistInfoItemExtractor {
|
|||
String getThumbnailUrl() throws ParsingException;
|
||||
String getPlaylistName() throws ParsingException;
|
||||
String getWebPageUrl() throws ParsingException;
|
||||
long getStreamsCount() throws ParsingException;
|
||||
long getStreamCount() throws ParsingException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
|
|||
try {
|
||||
result.resultList.add(streamCollector.extract(extractor));
|
||||
} catch (FoundAdException ae) {
|
||||
System.err.println("Found add");
|
||||
System.err.println("Found ad");
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
|
@ -69,7 +69,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
|
|||
try {
|
||||
result.resultList.add(channelCollector.extract(extractor));
|
||||
} catch (FoundAdException ae) {
|
||||
System.err.println("Found add");
|
||||
System.err.println("Found ad");
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,5 +49,5 @@ public abstract class SearchEngine {
|
|||
//Result search(String query, int page);
|
||||
public abstract InfoItemSearchCollector search(
|
||||
String query, int page, String contentCountry, EnumSet<Filter> filter)
|
||||
throws ExtractionException, IOException;
|
||||
throws IOException, ExtractionException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.InfoItem;
|
|||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 29.02.16.
|
||||
|
|
@ -31,7 +31,7 @@ import java.util.Vector;
|
|||
public class SearchResult {
|
||||
public static SearchResult getSearchResult(SearchEngine engine, String query,
|
||||
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
|
||||
throws ExtractionException, IOException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
SearchResult result = engine
|
||||
.search(query, page, languageCode, filter)
|
||||
|
|
@ -50,6 +50,6 @@ public class SearchResult {
|
|||
}
|
||||
|
||||
public String suggestion;
|
||||
public List<InfoItem> resultList = new Vector<>();
|
||||
public List<Throwable> errors = new Vector<>();
|
||||
public List<InfoItem> resultList = new ArrayList<>();
|
||||
public List<Throwable> errors = new ArrayList<>();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,37 +1,46 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class SoundcloudChannelExtractor extends ChannelExtractor {
|
||||
private String channelId;
|
||||
private JSONObject channel;
|
||||
private String nextUrl;
|
||||
|
||||
public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, url, serviceId);
|
||||
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
channelId = urlIdHandler.getId(url);
|
||||
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
channelId = getUrlIdHandler().getId(getOriginalUrl());
|
||||
String apiUrl = "https://api.soundcloud.com/users/" + channelId +
|
||||
"?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
channel = new JSONObject(response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
try {
|
||||
return channel.getString("permalink_url");
|
||||
} catch (Exception e) {
|
||||
return getOriginalUrl();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getChannelId() {
|
||||
return channelId;
|
||||
|
|
@ -56,32 +65,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=10"
|
||||
+ "&offset=0"
|
||||
+ "&linked_partitioning=1";
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
nextUrl = responseObject.getString("next_href")
|
||||
+ "&client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&linked_partitioning=1";
|
||||
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
JSONObject track = responseCollection.getJSONObject(i);
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
|
||||
}
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() {
|
||||
return channel.getLong("followers_count");
|
||||
|
|
@ -93,26 +76,27 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
|
||||
if (nextUrl.equals("")) {
|
||||
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/users/" + getChannelId() + "/tracks"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=20"
|
||||
+ "&linked_partitioning=1";
|
||||
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
|
||||
if (!hasMoreStreams()) {
|
||||
throw new ExtractionException("Channel doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
||||
|
||||
String response = dl.download(nextUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
nextUrl = responseObject.getString("next_href")
|
||||
+ "&client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&linked_partitioning=1";
|
||||
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
JSONObject track = responseCollection.getJSONObject(i);
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
|
||||
}
|
||||
return collector;
|
||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
|
|||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() {
|
||||
public long getStreamCount() {
|
||||
return searchResult.getLong("track_count");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -21,47 +18,28 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String getUrl(String channelId) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId());
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
return responseObject.getString("permalink_url");
|
||||
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + channelId);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(String siteUrl) throws ParsingException {
|
||||
public String getId(String url) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download(siteUrl);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element androidElement = doc.select("meta[property=al:android:url]").first();
|
||||
String id = androidElement.attr("content").substring(19);
|
||||
|
||||
return id;
|
||||
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String cleanUrl(String siteUrl) throws ParsingException {
|
||||
public String cleanUrl(String complexUrl) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
String response = dl.download(siteUrl);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element ogElement = doc.select("meta[property=og:url]").first();
|
||||
String url = ogElement.attr("content");
|
||||
|
||||
return url;
|
||||
return ogElement.attr("content");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
@ -71,6 +49,5 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
|
|||
public boolean acceptUrl(String channelUrl) {
|
||||
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
|
||||
return Parser.isMatch(regex, channelUrl.toLowerCase());
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,7 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
|
|
@ -15,61 +9,51 @@ import org.schabi.newpipe.extractor.Downloader;
|
|||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public class SoundcloudParsingHelper {
|
||||
private static String clientId;
|
||||
|
||||
private SoundcloudParsingHelper() {
|
||||
}
|
||||
|
||||
public static final String clientId() throws ReCaptchaException, IOException, RegexException {
|
||||
public static String clientId() throws ReCaptchaException, IOException, RegexException {
|
||||
if (clientId != null && !clientId.isEmpty()) return clientId;
|
||||
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download("https://soundcloud.com");
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
// TODO: Find a less heavy way to get the client_id
|
||||
// Currently we are downloading a 1MB file (!) just to get the client_id,
|
||||
// youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid.
|
||||
// The embed mode has a way to get it, but we still have to download a heavy file (~800KB).
|
||||
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
|
||||
String js = dl.download(jsElement.attr("src"));
|
||||
|
||||
String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
|
||||
clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
|
||||
return clientId;
|
||||
}
|
||||
|
||||
public static String toTimeAgoString(String time) throws ParsingException {
|
||||
try {
|
||||
List<Long> times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30),
|
||||
TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1),
|
||||
TimeUnit.SECONDS.toMillis(1));
|
||||
List<String> timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second");
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
|
||||
|
||||
long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime();
|
||||
|
||||
StringBuilder timeAgoString = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < times.size(); i++) {
|
||||
Long current = times.get(i);
|
||||
long currentAmount = timeAgo / current;
|
||||
if (currentAmount > 0) {
|
||||
timeAgoString.append(currentAmount).append(" ").append(timesString.get(i))
|
||||
.append(currentAmount != 1 ? "s ago" : " ago");
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (timeAgoString.toString().equals("")) {
|
||||
timeAgoString.append("Just now");
|
||||
}
|
||||
return timeAgoString.toString();
|
||||
} catch (ParseException e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public static String toDateString(String time) throws ParsingException {
|
||||
try {
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
|
||||
Date date = dateFormat.parse(time);
|
||||
Date date;
|
||||
// Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
|
||||
try {
|
||||
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
|
||||
} catch (Exception e) {
|
||||
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
|
||||
}
|
||||
|
||||
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
return newDateFormat.format(date);
|
||||
} catch (ParseException e) {
|
||||
|
|
@ -77,4 +61,83 @@ public class SoundcloudParsingHelper {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the endpoint "/resolve" of the api.<br/>
|
||||
* See https://developers.soundcloud.com/docs/api/reference#resolve
|
||||
*/
|
||||
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
|
||||
String apiUrl = "https://api.soundcloud.com/resolve"
|
||||
+ "?url=" + URLEncoder.encode(url, "UTF-8")
|
||||
+ "&client_id=" + clientId();
|
||||
|
||||
return new JSONObject(NewPipe.getDownloader().download(apiUrl));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).<br/>
|
||||
*
|
||||
* @return the url resolved
|
||||
*/
|
||||
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
||||
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
|
||||
+ URLEncoder.encode(apiUrl, "UTF-8"));
|
||||
|
||||
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the embed player with the url and return the id (like the id from the json api).<br/>
|
||||
*
|
||||
* @return the id resolved
|
||||
*/
|
||||
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
||||
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
|
||||
+ URLEncoder.encode(url, "UTF-8"));
|
||||
return Parser.matchGroup1(",\"id\":(.*?),", response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the streams from the given api and commit each of them to the collector.
|
||||
* <p>
|
||||
* This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always
|
||||
* get MIN_ITEMS or more items.
|
||||
*
|
||||
* @param minItems the method will return only when it have extracted that many items (equal or more)
|
||||
*/
|
||||
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
|
||||
|
||||
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl);
|
||||
}
|
||||
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the streams from the given api and commit each of them to the collector.
|
||||
*
|
||||
* @return the next streams url, empty if don't have
|
||||
*/
|
||||
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
String response = NewPipe.getDownloader().download(apiUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
|
||||
}
|
||||
|
||||
String nextStreamsUrl;
|
||||
try {
|
||||
nextStreamsUrl = responseObject.getString("next_href");
|
||||
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
|
||||
} catch (Exception ignored) {
|
||||
nextStreamsUrl = "";
|
||||
}
|
||||
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,38 +1,46 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
||||
private String playlistId;
|
||||
private JSONObject playlist;
|
||||
private List<String> nextTracks;
|
||||
|
||||
public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
|
||||
super(urlIdHandler, url, serviceId);
|
||||
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
playlistId = urlIdHandler.getId(url);
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
playlistId = getUrlIdHandler().getId(getOriginalUrl());
|
||||
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
|
||||
"?client_id=" + SoundcloudParsingHelper.clientId() +
|
||||
"&representation=compact";
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
playlist = new JSONObject(response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
try {
|
||||
return playlist.getString("permalink_url");
|
||||
} catch (Exception e) {
|
||||
return getOriginalUrl();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPlaylistId() {
|
||||
return playlistId;
|
||||
|
|
@ -69,61 +77,33 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long getStreamsCount() {
|
||||
public long getStreamCount() {
|
||||
return playlist.getLong("track_count");
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
// Note the "api", NOT "api-v2"
|
||||
String apiUrl = "https://api.soundcloud.com/playlists/" + getPlaylistId() + "/tracks"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=20"
|
||||
+ "&linked_partitioning=1";
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
JSONObject track = responseCollection.getJSONObject(i);
|
||||
try {
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
|
||||
} catch (Exception e) {
|
||||
nextTracks.add(track.getString("id"));
|
||||
}
|
||||
}
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException {
|
||||
if (nextTracks.equals(null)) {
|
||||
return null;
|
||||
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
|
||||
if (!hasMoreStreams()) {
|
||||
throw new ExtractionException("Playlist doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
||||
|
||||
// TODO: Do this per 10 tracks, instead of all tracks at once
|
||||
String apiUrl = "https://api-v2.soundcloud.com/tracks?ids=";
|
||||
for (String id : nextTracks) {
|
||||
apiUrl += id;
|
||||
if (!id.equals(nextTracks.get(nextTracks.size() - 1))) {
|
||||
apiUrl += ",";
|
||||
}
|
||||
}
|
||||
apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
JSONObject track = responseCollection.getJSONObject(i);
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
|
||||
}
|
||||
nextTracks = null;
|
||||
return collector;
|
||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -21,13 +18,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String getUrl(String listId) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId());
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
return responseObject.getString("permalink_url");
|
||||
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + listId);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
@ -36,15 +27,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String getId(String url) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download(url);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element androidElement = doc.select("meta[property=al:android:url]").first();
|
||||
String id = androidElement.attr("content").substring(23);
|
||||
|
||||
return id;
|
||||
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
@ -53,15 +36,10 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String cleanUrl(String complexUrl) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
String response = dl.download(complexUrl);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element ogElement = doc.select("meta[property=og:url]").first();
|
||||
String url = ogElement.attr("content");
|
||||
|
||||
return url;
|
||||
return ogElement.attr("content");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,5 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.EnumSet;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
|
|
@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.EnumSet;
|
||||
|
||||
public class SoundcloudSearchEngine extends SearchEngine {
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
|
|
|
|||
|
|
@ -13,61 +13,48 @@ import java.io.IOException;
|
|||
|
||||
public class SoundcloudService extends StreamingService {
|
||||
|
||||
public SoundcloudService(int id) {
|
||||
super(id);
|
||||
public SoundcloudService(int id, String name) {
|
||||
super(id, name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceInfo getServiceInfo() {
|
||||
ServiceInfo serviceInfo = new ServiceInfo();
|
||||
serviceInfo.name = "Soundcloud";
|
||||
return serviceInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamExtractor getStreamExtractorInstance(String url)
|
||||
throws ExtractionException, IOException {
|
||||
UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance();
|
||||
if (urlIdHandler.acceptUrl(url)) {
|
||||
return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId());
|
||||
} else {
|
||||
throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchEngine getSearchEngineInstance() {
|
||||
public SearchEngine getSearchEngine() {
|
||||
return new SoundcloudSearchEngine(getServiceId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getStreamUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getStreamUrlIdHandler() {
|
||||
return SoundcloudStreamUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getChannelUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getChannelUrlIdHandler() {
|
||||
return SoundcloudChannelUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getPlaylistUrlIdHandler() {
|
||||
return SoundcloudPlaylistUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
|
||||
return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
|
||||
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
|
||||
return new SoundcloudStreamExtractor(this, url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
|
||||
return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
|
||||
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return new SoundcloudChannelExtractor(this, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SuggestionExtractor getSuggestionExtractorInstance() {
|
||||
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SuggestionExtractor getSuggestionExtractor() {
|
||||
return new SoundcloudSuggestionExtractor(getServiceId());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
|
@ -21,33 +16,39 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
|||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
private String pageUrl;
|
||||
private String trackId;
|
||||
private JSONObject track;
|
||||
|
||||
public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, pageUrl, serviceId);
|
||||
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
super(service, url);
|
||||
}
|
||||
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
trackId = urlIdHandler.getId(pageUrl);
|
||||
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
track = new JSONObject(response);
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
|
||||
|
||||
if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
|
||||
throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
try {
|
||||
return track.getString("permalink_url");
|
||||
} catch (Exception e) {
|
||||
return getOriginalUrl();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId() {
|
||||
return trackId;
|
||||
return track.getInt("id") + "";
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -96,11 +97,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws ReCaptchaException, IOException, RegexException {
|
||||
Vector<AudioStream> audioStreams = new Vector<>();
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
List<AudioStream> audioStreams = new ArrayList<>();
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams"
|
||||
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + getId() + "/streams"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
|
|
@ -113,12 +114,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() {
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -126,7 +127,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
public int getTimeStamp() throws ParsingException {
|
||||
String timeStamp;
|
||||
try {
|
||||
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
|
||||
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
|
||||
} catch (Parser.RegexException e) {
|
||||
// catch this instantly since an url does not necessarily have to have a time stamp
|
||||
|
||||
|
|
@ -190,16 +191,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemExtractor getNextVideo() {
|
||||
public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related"
|
||||
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related"
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = dl.download(apiUrl);
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
|
|||
|
||||
@Override
|
||||
public String getUploadDate() throws ParsingException {
|
||||
return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at"));
|
||||
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -13,6 +10,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
|||
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
||||
|
||||
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
|
||||
|
||||
private SoundcloudStreamUrlIdHandler() {
|
||||
}
|
||||
|
||||
|
|
@ -23,13 +21,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String getUrl(String videoId) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId
|
||||
+ "?client_id=" + SoundcloudParsingHelper.clientId());
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
return responseObject.getString("permalink_url");
|
||||
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + videoId);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
@ -38,15 +30,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String getId(String url) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String response = dl.download(url);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element androidElement = doc.select("meta[property=al:android:url]").first();
|
||||
String id = androidElement.attr("content").substring(20);
|
||||
|
||||
return id;
|
||||
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
@ -55,15 +39,10 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
|||
@Override
|
||||
public String cleanUrl(String complexUrl) throws ParsingException {
|
||||
try {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
String response = dl.download(complexUrl);
|
||||
Document doc = Jsoup.parse(response);
|
||||
|
||||
Element ogElement = doc.select("meta[property=og:url]").first();
|
||||
String url = ogElement.attr("content");
|
||||
|
||||
return url;
|
||||
return ogElement.attr("content");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,5 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
|
|
@ -13,6 +8,11 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
|
|||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
|
||||
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ public class ItagItem {
|
|||
// Disable Opus codec as it's not well supported in older devices
|
||||
// new ItagItem(249, AUDIO, WEBMA, 50),
|
||||
// new ItagItem(250, AUDIO, WEBMA, 70),
|
||||
// new ItagItem(251, AUDIO, WEBMA, 16),
|
||||
// new ItagItem(251, AUDIO, WEBMA, 160),
|
||||
new ItagItem(171, AUDIO, WEBMA, 128),
|
||||
new ItagItem(172, AUDIO, WEBMA, 256),
|
||||
new ItagItem(139, AUDIO, M4A, 48),
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import org.jsoup.nodes.Document;
|
|||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -44,6 +44,7 @@ import java.io.IOException;
|
|||
@SuppressWarnings("WeakerAccess")
|
||||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
|
||||
|
||||
private Document doc;
|
||||
/**
|
||||
|
|
@ -51,29 +52,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
*/
|
||||
private Document nextStreamsAjax;
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Variables for cache purposes (not "select" the current document all over again)
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
private String channelId;
|
||||
private String channelName;
|
||||
private String avatarUrl;
|
||||
private String bannerUrl;
|
||||
private String feedUrl;
|
||||
private long subscriberCount = -1;
|
||||
public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
|
||||
fetchDocument();
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
String userUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS;
|
||||
String pageContent = downloader.download(userUrl);
|
||||
doc = Jsoup.parse(pageContent, userUrl);
|
||||
|
||||
nextStreamsUrl = getNextStreamsUrlFrom(doc);
|
||||
nextStreamsAjax = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getChannelId() throws ParsingException {
|
||||
try {
|
||||
if (channelId == null) {
|
||||
channelId = getUrlIdHandler().getId(getUrl());
|
||||
}
|
||||
|
||||
return channelId;
|
||||
return getUrlIdHandler().getId(getCleanUrl());
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get channel id");
|
||||
}
|
||||
|
|
@ -82,11 +80,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Override
|
||||
public String getChannelName() throws ParsingException {
|
||||
try {
|
||||
if (channelName == null) {
|
||||
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
|
||||
}
|
||||
|
||||
return channelName;
|
||||
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get channel name");
|
||||
}
|
||||
|
|
@ -95,11 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Override
|
||||
public String getAvatarUrl() throws ParsingException {
|
||||
try {
|
||||
if (avatarUrl == null) {
|
||||
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
|
||||
}
|
||||
|
||||
return avatarUrl;
|
||||
return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get avatar", e);
|
||||
}
|
||||
|
|
@ -108,59 +98,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Override
|
||||
public String getBannerUrl() throws ParsingException {
|
||||
try {
|
||||
if (bannerUrl == null) {
|
||||
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
|
||||
String cssContent = el.html();
|
||||
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
|
||||
|
||||
bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
|
||||
}
|
||||
|
||||
return bannerUrl;
|
||||
return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get Banner", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getStreams() throws ParsingException {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
|
||||
collectStreamsFrom(collector, ul);
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
|
||||
if (subscriberCount == -1) {
|
||||
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||
if (el != null) {
|
||||
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||
} else {
|
||||
throw new ParsingException("Could not get subscriber count");
|
||||
}
|
||||
}
|
||||
|
||||
return subscriberCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFeedUrl() throws ParsingException {
|
||||
try {
|
||||
if (feedUrl == null) {
|
||||
String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
|
||||
feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
|
||||
}
|
||||
|
||||
return feedUrl;
|
||||
return channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get feed url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
|
||||
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
|
||||
collectStreamsFrom(collector, ul);
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
|
||||
if (!hasMoreStreams()) {
|
||||
throw new ExtractionException("Channel doesn't have more streams");
|
||||
}
|
||||
|
|
@ -169,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
||||
|
||||
return collector;
|
||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
||||
}
|
||||
|
||||
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
|
@ -182,8 +160,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
|
||||
if (!nextStreamsHtmlDataRaw.isEmpty()) {
|
||||
Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
|
||||
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
|
||||
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
|
||||
} else {
|
||||
nextStreamsUrl = "";
|
||||
}
|
||||
|
|
@ -192,7 +169,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private String getNextStreamsUrl(Document d) throws ParsingException {
|
||||
private String getNextStreamsUrlFrom(Document d) throws ParsingException {
|
||||
try {
|
||||
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
|
||||
if (button != null) {
|
||||
|
|
@ -206,17 +183,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000";
|
||||
String pageContent = downloader.download(userUrl);
|
||||
doc = Jsoup.parse(pageContent, userUrl);
|
||||
|
||||
nextStreamsUrl = getNextStreamsUrl(doc);
|
||||
nextStreamsAjax = null;
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
|
||||
collector.getItemList().clear();
|
||||
|
||||
|
|
@ -230,7 +196,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
|
||||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -259,7 +226,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
public int getDuration() throws ParsingException {
|
||||
try {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
li.select("span[class=\"video-time\"]").first().text());
|
||||
li.select("span[class*=\"video-time\"]").first().text());
|
||||
} catch (Exception e) {
|
||||
if (isLiveStream(li)) {
|
||||
// -1 for no duration
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
|||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
public long getStreamCount() throws ParsingException {
|
||||
Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
|
||||
if (metaEl == null) {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import org.jsoup.nodes.Document;
|
|||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -23,39 +24,31 @@ import java.io.IOException;
|
|||
@SuppressWarnings("WeakerAccess")
|
||||
public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
private Document doc = null;
|
||||
private Document doc;
|
||||
/**
|
||||
* It's lazily initialized (when getNextStreams is called)
|
||||
*/
|
||||
private Document nextStreamsAjax = null;
|
||||
private Document nextStreamsAjax;
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Variables for cache purposes (not "select" the current document all over again)
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
private String playlistId;
|
||||
private String playlistName;
|
||||
private String avatarUrl;
|
||||
private String bannerUrl;
|
||||
public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
private long streamsCount;
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
private String uploaderUrl;
|
||||
private String uploaderName;
|
||||
private String uploaderAvatarUrl;
|
||||
String pageContent = downloader.download(getCleanUrl());
|
||||
doc = Jsoup.parse(pageContent, getCleanUrl());
|
||||
|
||||
public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
|
||||
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
|
||||
fetchDocument();
|
||||
nextStreamsUrl = getNextStreamsUrlFrom(doc);
|
||||
nextStreamsAjax = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPlaylistId() throws ParsingException {
|
||||
try {
|
||||
if (playlistId == null) {
|
||||
playlistId = getUrlIdHandler().getId(getUrl());
|
||||
}
|
||||
|
||||
return playlistId;
|
||||
return getUrlIdHandler().getId(getCleanUrl());
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist id");
|
||||
}
|
||||
|
|
@ -64,11 +57,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getPlaylistName() throws ParsingException {
|
||||
try {
|
||||
if (playlistName == null) {
|
||||
playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
|
||||
}
|
||||
|
||||
return playlistName;
|
||||
return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist name");
|
||||
}
|
||||
|
|
@ -77,11 +66,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getAvatarUrl() throws ParsingException {
|
||||
try {
|
||||
if (avatarUrl == null) {
|
||||
avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
|
||||
}
|
||||
|
||||
return avatarUrl;
|
||||
return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist avatar");
|
||||
}
|
||||
|
|
@ -90,18 +75,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getBannerUrl() throws ParsingException {
|
||||
try {
|
||||
if (bannerUrl == null) {
|
||||
Element el = doc.select("div[id=\"gh-banner\"] style").first();
|
||||
String cssContent = el.html();
|
||||
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
|
||||
if (url.contains("s.ytimg.com")) {
|
||||
bannerUrl = null;
|
||||
return null;
|
||||
} else {
|
||||
bannerUrl = url.substring(0, url.indexOf(");"));
|
||||
}
|
||||
return url.substring(0, url.indexOf(");"));
|
||||
}
|
||||
|
||||
return bannerUrl;
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist Banner");
|
||||
}
|
||||
|
|
@ -110,11 +93,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
try {
|
||||
if (uploaderUrl == null) {
|
||||
uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
|
||||
}
|
||||
|
||||
return uploaderUrl;
|
||||
return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader name");
|
||||
}
|
||||
|
|
@ -123,11 +102,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
try {
|
||||
if (uploaderName == null) {
|
||||
uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
|
||||
}
|
||||
|
||||
return uploaderName;
|
||||
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader name");
|
||||
}
|
||||
|
|
@ -136,19 +111,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getUploaderAvatarUrl() throws ParsingException {
|
||||
try {
|
||||
if (uploaderAvatarUrl == null) {
|
||||
uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
|
||||
}
|
||||
|
||||
return uploaderAvatarUrl;
|
||||
return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader avatar");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamsCount() throws ParsingException {
|
||||
if (streamsCount <= 0) {
|
||||
public long getStreamCount() throws ParsingException {
|
||||
String input;
|
||||
|
||||
try {
|
||||
|
|
@ -158,32 +128,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
|
||||
try {
|
||||
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(input));
|
||||
} catch (NumberFormatException e) {
|
||||
// When there's no videos in a playlist, there's no number in the "innerHtml",
|
||||
// all characters that is not a number is removed, so we try to parse a empty string
|
||||
if (!input.isEmpty()) {
|
||||
streamsCount = 0;
|
||||
return 0;
|
||||
} else {
|
||||
throw new ParsingException("Could not handle input: " + input, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return streamsCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getStreams() throws ParsingException {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
|
||||
collectStreamsFrom(collector, tbody);
|
||||
return collector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
|
||||
if (!hasMoreStreams()){
|
||||
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
|
||||
if (!hasMoreStreams()) {
|
||||
throw new ExtractionException("Playlist doesn't have more streams");
|
||||
}
|
||||
|
||||
|
|
@ -191,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||
|
||||
return collector;
|
||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
||||
}
|
||||
|
||||
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
|
@ -204,8 +171,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
|
||||
if (!nextStreamsHtmlDataRaw.isEmpty()) {
|
||||
final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
|
||||
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
|
||||
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
|
||||
} else {
|
||||
nextStreamsUrl = "";
|
||||
}
|
||||
|
|
@ -214,17 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
String pageContent = downloader.download(getUrl());
|
||||
doc = Jsoup.parse(pageContent, getUrl());
|
||||
|
||||
nextStreamsUrl = getNextStreamsUrl(doc);
|
||||
nextStreamsAjax = null;
|
||||
}
|
||||
|
||||
private String getNextStreamsUrl(Document d) throws ParsingException {
|
||||
private String getNextStreamsUrlFrom(Document d) throws ParsingException {
|
||||
try {
|
||||
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
|
||||
if (button != null) {
|
||||
|
|
@ -241,7 +197,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
|
||||
collector.getItemList().clear();
|
||||
|
||||
final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
|
||||
final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
|
||||
for (final Element li : element.children()) {
|
||||
collector.commit(new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
|
|
@ -252,7 +208,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getWebPageUrl() throws ParsingException {
|
||||
try {
|
||||
return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id"));
|
||||
return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get web page url for the video", e);
|
||||
}
|
||||
|
|
@ -300,7 +256,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
|
||||
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,61 +34,48 @@ import java.io.IOException;
|
|||
|
||||
public class YoutubeService extends StreamingService {
|
||||
|
||||
public YoutubeService(int id) {
|
||||
super(id);
|
||||
public YoutubeService(int id, String name) {
|
||||
super(id, name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceInfo getServiceInfo() {
|
||||
ServiceInfo serviceInfo = new ServiceInfo();
|
||||
serviceInfo.name = "Youtube";
|
||||
return serviceInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamExtractor getStreamExtractorInstance(String url)
|
||||
throws ExtractionException, IOException {
|
||||
UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
|
||||
if (urlIdHandler.acceptUrl(url)) {
|
||||
return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
|
||||
} else {
|
||||
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchEngine getSearchEngineInstance() {
|
||||
public SearchEngine getSearchEngine() {
|
||||
return new YoutubeSearchEngine(getServiceId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getStreamUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getStreamUrlIdHandler() {
|
||||
return YoutubeStreamUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getChannelUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getChannelUrlIdHandler() {
|
||||
return YoutubeChannelUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
|
||||
public UrlIdHandler getPlaylistUrlIdHandler() {
|
||||
return YoutubePlaylistUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
|
||||
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
|
||||
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
|
||||
return new YoutubeStreamExtractor(this, url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
|
||||
return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
|
||||
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return new YoutubeChannelExtractor(this, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SuggestionExtractor getSuggestionExtractorInstance() {
|
||||
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
return new YoutubePlaylistExtractor(this, url, nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SuggestionExtractor getSuggestionExtractor() {
|
||||
return new YoutubeSuggestionExtractor(getServiceId());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import org.mozilla.javascript.Function;
|
|||
import org.mozilla.javascript.ScriptableObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
|
@ -26,9 +26,9 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
|||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Vector;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -80,12 +80,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
/*//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private Document doc;
|
||||
private final String dirtyUrl;
|
||||
|
||||
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
|
||||
dirtyUrl = pageUrl;
|
||||
fetchDocument();
|
||||
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
super(service, url);
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -95,7 +92,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
try {
|
||||
return getUrlIdHandler().getId(getUrl());
|
||||
return getUrlIdHandler().getId(getCleanUrl());
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get stream id");
|
||||
}
|
||||
|
|
@ -238,8 +235,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws ParsingException {
|
||||
Vector<AudioStream> audioStreams = new Vector<>();
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
List<AudioStream> audioStreams = new ArrayList<>();
|
||||
try {
|
||||
String encodedUrlMap;
|
||||
// playerArgs could be null if the video is age restricted
|
||||
|
|
@ -288,8 +285,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws ParsingException {
|
||||
Vector<VideoStream> videoStreams = new Vector<>();
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
String encodedUrlMap;
|
||||
|
|
@ -342,8 +339,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() throws ParsingException {
|
||||
Vector<VideoStream> videoOnlyStreams = new Vector<>();
|
||||
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
|
||||
List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
String encodedUrlMap;
|
||||
|
|
@ -405,7 +402,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public int getTimeStamp() throws ParsingException {
|
||||
String timeStamp;
|
||||
try {
|
||||
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
|
||||
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
|
||||
} catch (Parser.RegexException e) {
|
||||
// catch this instantly since an url does not necessarily have to have a time stamp
|
||||
|
||||
|
|
@ -516,7 +513,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemExtractor getNextVideo() throws ParsingException {
|
||||
public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
|
||||
try {
|
||||
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
|
||||
.select("li").first());
|
||||
|
|
@ -526,9 +523,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getRelatedVideos() throws ParsingException {
|
||||
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
|
||||
try {
|
||||
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
Element ul = doc.select("ul[id=\"watch-related\"]").first();
|
||||
if (ul != null) {
|
||||
for (Element li : ul.children()) {
|
||||
|
|
@ -617,11 +614,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
// cached values
|
||||
private static volatile String decryptionCode = "";
|
||||
|
||||
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
|
||||
@Override
|
||||
public void fetchPage() throws IOException, ExtractionException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
String pageContent = downloader.download(getUrl());
|
||||
doc = Jsoup.parse(pageContent, getUrl());
|
||||
String pageContent = downloader.download(getCleanUrl());
|
||||
doc = Jsoup.parse(pageContent, getCleanUrl());
|
||||
|
||||
JSONObject ytPlayerConfig;
|
||||
String playerUrl;
|
||||
|
|
@ -632,7 +630,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
// Check if the video is age restricted
|
||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
|
||||
playerUrl = getPlayerUrlFromRestrictedVideo();
|
||||
isAgeRestricted = true;
|
||||
} else {
|
||||
ytPlayerConfig = getPlayerConfig(pageContent);
|
||||
|
|
@ -683,7 +681,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
if (isLiveStream) {
|
||||
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
|
||||
throw new LiveStreamException("This is a Live stream. Can't use those right now.");
|
||||
}
|
||||
|
||||
return playerArgs;
|
||||
|
|
@ -709,12 +707,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
|
||||
private String getPlayerUrlFromRestrictedVideo() throws ParsingException, ReCaptchaException {
|
||||
try {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
String playerUrl = "";
|
||||
String videoId = getUrlIdHandler().getId(pageUrl);
|
||||
String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||
String embedUrl = "https://www.youtube.com/embed/" + getId();
|
||||
String embedPageContent = downloader.download(embedUrl);
|
||||
//todo: find out if this can be reapaced by Parser.matchGroup1()
|
||||
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
|
||||
|
|
@ -811,7 +808,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
|
||||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -829,8 +827,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public int getDuration() throws ParsingException {
|
||||
try {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
li.select("span.video-time").first().text());
|
||||
li.select("span[class*=\"video-time\"]").first().text());
|
||||
} catch (Exception e) {
|
||||
if (isLiveStream(li)) {
|
||||
// -1 for no duration
|
||||
return -1;
|
||||
} else {
|
||||
throw new ParsingException("Could not get Duration: " + getTitle(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -845,12 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
//this line is unused
|
||||
//String views = li.select("span.view-count").first().text();
|
||||
|
||||
//Log.i(TAG, "title:"+info.title);
|
||||
//Log.i(TAG, "view count:"+views);
|
||||
|
||||
try {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(
|
||||
li.select("span.view-count").first().text()));
|
||||
|
|
@ -875,6 +876,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
return thumbnailUrl;
|
||||
}
|
||||
|
||||
private boolean isLiveStream(Element item) {
|
||||
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
|
||||
|
||||
if (bla == null) {
|
||||
// sometimes livestreams dont have badges but sill are live streams
|
||||
// if video time is not available we most likly have an offline livestream
|
||||
if (item.select("span[class*=\"video-time\"]").first() == null) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return bla != null;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
|||
public int getDuration() throws ParsingException {
|
||||
try {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
item.select("span[class=\"video-time\"]").first().text());
|
||||
item.select("span[class*=\"video-time\"]").first().text());
|
||||
} catch (Exception e) {
|
||||
if (isLiveStream(item)) {
|
||||
// -1 for no duration
|
||||
|
|
@ -104,16 +104,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
|||
if (div == null) {
|
||||
return -1;
|
||||
} else {
|
||||
input = div.select("li").get(1)
|
||||
.text();
|
||||
input = div.select("li").get(1).text();
|
||||
}
|
||||
} catch (IndexOutOfBoundsException e) {
|
||||
if (isLiveStream(item)) {
|
||||
// -1 for no view count
|
||||
return -1;
|
||||
} else {
|
||||
throw new ParsingException(
|
||||
"Could not parse yt-lockup-meta although available: " + getTitle(), e);
|
||||
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getTitle(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -161,7 +159,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
|||
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return !item.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||
return !item.select("span[class*=\"icon-not-available\"]").isEmpty() ||
|
||||
!item.select("span[class*=\"yt-badge-ad\"]").isEmpty();
|
||||
}
|
||||
|
||||
private boolean isLiveStream(Element item) {
|
||||
|
|
|
|||
|
|
@ -1,25 +1,17 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.SuggestionExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 28.09.16.
|
||||
*
|
||||
|
|
@ -49,51 +41,24 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<String> suggestionList(
|
||||
String query, String contentCountry)
|
||||
throws ExtractionException, IOException {
|
||||
public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
List<String> suggestions = new ArrayList<>();
|
||||
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String url = "https://suggestqueries.google.com/complete/search"
|
||||
+ "?client=" + ""
|
||||
+ "&output=" + "toolbar"
|
||||
+ "?client=" + "firefox" // 'toolbar' for xml
|
||||
+ "&ds=" + "yt"
|
||||
+ "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
|
||||
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
|
||||
|
||||
|
||||
String response = dl.download(url);
|
||||
|
||||
//TODO: Parse xml data using Jsoup not done
|
||||
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder dBuilder;
|
||||
org.w3c.dom.Document doc = null;
|
||||
|
||||
try {
|
||||
dBuilder = dbFactory.newDocumentBuilder();
|
||||
doc = dBuilder.parse(new InputSource(
|
||||
new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8))));
|
||||
doc.getDocumentElement().normalize();
|
||||
} catch (ParserConfigurationException | SAXException | IOException e) {
|
||||
throw new ParsingException("Could not parse document.");
|
||||
}
|
||||
|
||||
try {
|
||||
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
|
||||
for (int temp = 0; temp < nList.getLength(); temp++) {
|
||||
|
||||
NodeList nList1 = doc.getElementsByTagName("suggestion");
|
||||
Node nNode1 = nList1.item(temp);
|
||||
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
|
||||
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
|
||||
suggestions.add(eElement.getAttribute("data"));
|
||||
}
|
||||
}
|
||||
return suggestions;
|
||||
JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1);
|
||||
for (Object suggestion : suggestionsArray) suggestions.add(suggestion.toString());
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get suggestions form document.", e);
|
||||
throw new ParsingException("Could not parse suggestions response.", e);
|
||||
}
|
||||
|
||||
return suggestions;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ public abstract class Stream implements Serializable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Reveals whether two streams are the same, but have different urls
|
||||
* Reveals whether two streams have the same stats (format and bitrate, for example)
|
||||
*/
|
||||
public boolean equalStats(Stream cmp) {
|
||||
return cmp != null && format == cmp.format;
|
||||
|
|
|
|||
|
|
@ -21,9 +21,10 @@ package org.schabi.newpipe.extractor.stream;
|
|||
*/
|
||||
|
||||
import org.schabi.newpipe.extractor.Extractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
|
@ -33,8 +34,14 @@ import java.util.List;
|
|||
*/
|
||||
public abstract class StreamExtractor extends Extractor {
|
||||
|
||||
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
|
||||
super(urlIdHandler, serviceId, url);
|
||||
public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
super(service, url);
|
||||
fetchPage();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
|
||||
return getService().getStreamUrlIdHandler();
|
||||
}
|
||||
|
||||
public abstract String getId() throws ParsingException;
|
||||
|
|
@ -48,16 +55,16 @@ public abstract class StreamExtractor extends Extractor {
|
|||
public abstract String getUploadDate() throws ParsingException;
|
||||
public abstract String getThumbnailUrl() throws ParsingException;
|
||||
public abstract String getUploaderThumbnailUrl() throws ParsingException;
|
||||
public abstract List<AudioStream> getAudioStreams() throws ParsingException, ReCaptchaException, IOException;
|
||||
public abstract List<VideoStream> getVideoStreams() throws ParsingException;
|
||||
public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException;
|
||||
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
|
||||
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
|
||||
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
|
||||
public abstract String getDashMpdUrl() throws ParsingException;
|
||||
public abstract int getAgeLimit() throws ParsingException;
|
||||
public abstract String getAverageRating() throws ParsingException;
|
||||
public abstract int getLikeCount() throws ParsingException;
|
||||
public abstract int getDislikeCount() throws ParsingException;
|
||||
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
|
||||
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException;
|
||||
public abstract StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException;
|
||||
public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException;
|
||||
public abstract StreamType getStreamType() throws ParsingException;
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.stream;
|
|||
|
||||
import org.schabi.newpipe.extractor.Info;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.utils.DashMpdParser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 26.08.15.
|
||||
|
|
@ -46,11 +50,23 @@ public class StreamInfo extends Info {
|
|||
public StreamInfo() {
|
||||
}
|
||||
|
||||
public static StreamInfo getInfo(String url) throws IOException, ExtractionException {
|
||||
return getInfo(NewPipe.getServiceByUrl(url), url);
|
||||
}
|
||||
|
||||
public static StreamInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
|
||||
return getInfo(serviceItem.getService(), url);
|
||||
}
|
||||
|
||||
public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
|
||||
return getInfo(service.getStreamExtractor(url));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills out the video info fields which are common to all services.
|
||||
* Probably needs to be overridden by subclasses
|
||||
*/
|
||||
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
|
||||
public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
|
||||
StreamInfo streamInfo = new StreamInfo();
|
||||
|
||||
try {
|
||||
|
|
@ -80,7 +96,7 @@ public class StreamInfo extends Info {
|
|||
// if one of these is not available an exception is meant to be thrown directly into the frontend.
|
||||
|
||||
streamInfo.service_id = extractor.getServiceId();
|
||||
streamInfo.url = extractor.getUrl();
|
||||
streamInfo.url = extractor.getCleanUrl();
|
||||
streamInfo.stream_type = extractor.getStreamType();
|
||||
streamInfo.id = extractor.getId();
|
||||
streamInfo.name = extractor.getTitle();
|
||||
|
|
@ -128,15 +144,12 @@ public class StreamInfo extends Info {
|
|||
}
|
||||
|
||||
// Lists can be null if a exception was thrown during extraction
|
||||
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
|
||||
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
|
||||
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
|
||||
if (streamInfo.video_streams == null) streamInfo.video_streams = new ArrayList<>();
|
||||
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>();
|
||||
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>();
|
||||
|
||||
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
|
||||
try {
|
||||
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
|
||||
// it has video, video only and audio streams and will only add to the list if it don't
|
||||
// find a similar stream in the respective lists (calling Stream#equalStats).
|
||||
DashMpdParser.getStreams(streamInfo);
|
||||
} catch (Exception e) {
|
||||
// Sometimes we receive 403 (forbidden) error when trying to download the manifest,
|
||||
|
|
@ -246,6 +259,8 @@ public class StreamInfo extends Info {
|
|||
streamInfo.addException(e);
|
||||
}
|
||||
|
||||
if (streamInfo.related_streams == null) streamInfo.related_streams = new ArrayList<>();
|
||||
|
||||
return streamInfo;
|
||||
}
|
||||
|
||||
|
|
@ -278,7 +293,7 @@ public class StreamInfo extends Info {
|
|||
public int dislike_count = -1;
|
||||
public String average_rating;
|
||||
public StreamInfoItem next_video;
|
||||
public List<InfoItem> related_streams = new Vector<>();
|
||||
public List<InfoItem> related_streams;
|
||||
//in seconds. some metadata is not passed using a StreamInfo object!
|
||||
public int start_position = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,13 @@ public class DashMpdParser {
|
|||
}
|
||||
|
||||
/**
|
||||
* Download manifest and return nodelist with elements of tag "AdaptationSet"
|
||||
* Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
|
||||
* then it will search for any stream that the ItagItem has (by the id).
|
||||
* <p>
|
||||
* It has video, video only and audio streams and will only add to the list if it don't
|
||||
* find a similar stream in the respective lists (calling {@link Stream#equalStats}).
|
||||
*
|
||||
* @param streamInfo where the parsed streams will be added
|
||||
*/
|
||||
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
|
||||
String dashDoc;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ public class Utils {
|
|||
* Remove all non-digit characters from a string.<p>
|
||||
* Examples:<br/>
|
||||
* <ul><li>1 234 567 views -> 1234567</li>
|
||||
* <li>$ 31,133.124 -> 31133124</li></ul>
|
||||
* <li>$31,133.124 -> 31133124</li></ul>
|
||||
*
|
||||
* @param toRemove string to remove non-digit chars
|
||||
* @return a string that contains only digits
|
||||
|
|
@ -21,17 +21,25 @@ public class Utils {
|
|||
/**
|
||||
* Check if throwable have the cause
|
||||
*/
|
||||
public static boolean hasCauseThrowable(Throwable throwable, Class<?> causeToCheck) {
|
||||
public static boolean hasCauseThrowable(Throwable throwable, Class<?>... causesToCheck) {
|
||||
// Check if getCause is not the same as cause (the getCause is already the root),
|
||||
// as it will cause a infinite loop if it is
|
||||
Throwable cause, getCause = throwable;
|
||||
|
||||
for (Class<?> causesEl : causesToCheck) {
|
||||
if (throwable.getClass().isAssignableFrom(causesEl)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
while ((cause = throwable.getCause()) != null && getCause != cause) {
|
||||
getCause = cause;
|
||||
if (cause.getClass().isAssignableFrom(causeToCheck)) {
|
||||
for (Class<?> causesEl : causesToCheck) {
|
||||
if (cause.getClass().isAssignableFrom(causesEl)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import java.util.Map;
|
|||
import javax.net.ssl.HttpsURLConnection;
|
||||
|
||||
|
||||
/**
|
||||
/*
|
||||
* Created by Christian Schabesberger on 28.01.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -41,10 +41,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
|
||||
private static Downloader instance = null;
|
||||
|
||||
private Downloader() {}
|
||||
private Downloader() {
|
||||
}
|
||||
|
||||
public static Downloader getInstance() {
|
||||
if(instance == null) {
|
||||
if (instance == null) {
|
||||
synchronized (Downloader.class) {
|
||||
if (instance == null) {
|
||||
instance = new Downloader();
|
||||
|
|
@ -62,11 +63,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
return Downloader.mCookies;
|
||||
}
|
||||
|
||||
/**Download the text file at the supplied URL as in download(String),
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String),
|
||||
* but set the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param language the language (usually a 2-character code) to set as the preferred language
|
||||
* @return the contents of the specified text file*/
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
|
||||
Map<String, String> requestProperties = new HashMap<>();
|
||||
requestProperties.put("Accept-Language", language);
|
||||
|
|
@ -74,29 +78,35 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
}
|
||||
|
||||
|
||||
/**Download the text file at the supplied URL as in download(String),
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String),
|
||||
* but set the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param customProperties set request header properties
|
||||
* @return the contents of the specified text file
|
||||
* @throws IOException*/
|
||||
* @throws IOException
|
||||
*/
|
||||
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
Iterator it = customProperties.entrySet().iterator();
|
||||
while(it.hasNext()) {
|
||||
Map.Entry pair = (Map.Entry)it.next();
|
||||
con.setRequestProperty((String)pair.getKey(), (String)pair.getValue());
|
||||
while (it.hasNext()) {
|
||||
Map.Entry pair = (Map.Entry) it.next();
|
||||
con.setRequestProperty((String) pair.getKey(), (String) pair.getValue());
|
||||
}
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
/**Common functionality between download(String url) and download(String url, String language)*/
|
||||
/**
|
||||
* Common functionality between download(String url) and download(String url, String language)
|
||||
*/
|
||||
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
|
||||
StringBuilder response = new StringBuilder();
|
||||
BufferedReader in = null;
|
||||
|
||||
try {
|
||||
con.setReadTimeout(30 * 1000);// 30s
|
||||
con.setRequestMethod("GET");
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
|
||||
|
|
@ -108,13 +118,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
new InputStreamReader(con.getInputStream()));
|
||||
String inputLine;
|
||||
|
||||
while((inputLine = in.readLine()) != null) {
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
|
||||
} catch (UnknownHostException uhe) {//thrown when there's no internet connection
|
||||
throw new IOException("unknown host or no network", uhe);
|
||||
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
|
||||
} catch(Exception e) {
|
||||
} catch (Exception e) {
|
||||
/*
|
||||
* HTTP 429 == Too Many Request
|
||||
* Receive from Youtube.com = ReCaptcha challenge request
|
||||
|
|
@ -123,9 +133,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
}
|
||||
throw new IOException(e);
|
||||
|
||||
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
|
||||
} finally {
|
||||
if(in != null) {
|
||||
if (in != null) {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -133,10 +144,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
|||
return response.toString();
|
||||
}
|
||||
|
||||
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
|
||||
/**
|
||||
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
|
||||
* Primarily intended for downloading web pages.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to download
|
||||
* @return the contents of the specified text file*/
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashSet;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
|
||||
|
||||
public class NewPipeTest {
|
||||
@Test
|
||||
public void getAllServicesTest() throws Exception {
|
||||
assertEquals(NewPipe.getServices().length, ServiceList.values().length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAllServicesHaveDifferentId() throws Exception {
|
||||
HashSet<Integer> servicesId = new HashSet<>();
|
||||
for (StreamingService streamingService : NewPipe.getServices()) {
|
||||
String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().name + ")";
|
||||
|
||||
assertTrue(errorMsg, servicesId.add(streamingService.getServiceId()));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getServiceWithId() throws Exception {
|
||||
assertEquals(NewPipe.getService(Youtube.getId()), Youtube.getService());
|
||||
assertEquals(NewPipe.getService(SoundCloud.getId()), SoundCloud.getService());
|
||||
|
||||
assertNotEquals(NewPipe.getService(SoundCloud.getId()), Youtube.getService());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getServiceWithName() throws Exception {
|
||||
assertEquals(NewPipe.getService(Youtube.getServiceInfo().name), Youtube.getService());
|
||||
assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().name), SoundCloud.getService());
|
||||
|
||||
assertNotEquals(NewPipe.getService(Youtube.getServiceInfo().name), SoundCloud.getService());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getServiceWithUrl() throws Exception {
|
||||
assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), Youtube.getService());
|
||||
assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), Youtube.getService());
|
||||
assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), Youtube.getService());
|
||||
assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud.getService());
|
||||
assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud.getService());
|
||||
assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud.getService());
|
||||
|
||||
assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), Youtube.getService());
|
||||
assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud.getService());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getIdWithServiceName() throws Exception {
|
||||
assertEquals(NewPipe.getIdOfService(Youtube.getServiceInfo().name), Youtube.getId());
|
||||
assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), SoundCloud.getId());
|
||||
|
||||
assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), Youtube.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getServiceNameWithId() throws Exception {
|
||||
assertEquals(NewPipe.getNameOfService(Youtube.getId()), Youtube.getServiceInfo().name);
|
||||
assertEquals(NewPipe.getNameOfService(SoundCloud.getId()), SoundCloud.getServiceInfo().name);
|
||||
|
||||
assertNotEquals(NewPipe.getNameOfService(Youtube.getId()), SoundCloud.getServiceInfo().name);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +1,17 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static junit.framework.Assert.assertNotNull;
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
|
||||
/**
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.09.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -41,8 +42,8 @@ public class YoutubeChannelExtractorTest {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = NewPipe.getService("Youtube")
|
||||
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
|
||||
extractor = Youtube.getService()
|
||||
.getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -61,7 +62,7 @@ public class YoutubeChannelExtractorTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetBannerurl() throws Exception {
|
||||
public void testGetBannerUrl() throws Exception {
|
||||
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3"));
|
||||
}
|
||||
|
||||
|
|
@ -81,9 +82,10 @@ public class YoutubeChannelExtractorTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testHasNextPage() throws Exception {
|
||||
// this particular example (link) has a next page !!!
|
||||
assertTrue("no next page link found", extractor.hasMoreStreams());
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
assertTrue("don't have more streams", extractor.hasMoreStreams());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -92,16 +94,11 @@ public class YoutubeChannelExtractorTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetNextPage() throws Exception {
|
||||
extractor = NewPipe.getService("Youtube")
|
||||
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
|
||||
assertTrue("next page didn't have content", !extractor.getStreams().getItemList().isEmpty());
|
||||
public void testGetNextStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
|
||||
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNextNextPageUrl() throws Exception {
|
||||
extractor = NewPipe.getService("Youtube")
|
||||
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
|
||||
assertTrue("next page didn't have content", extractor.hasMoreStreams());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/**
|
||||
* Test for {@link PlaylistExtractor}
|
||||
*/
|
||||
|
||||
public class YoutubePlaylistExtractorTest {
|
||||
private PlaylistExtractor extractor;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = Youtube.getService()
|
||||
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDownloader() throws Exception {
|
||||
assertNotNull(NewPipe.getDownloader());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetId() throws Exception {
|
||||
assertEquals(extractor.getPlaylistId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetName() throws Exception {
|
||||
assertEquals(extractor.getPlaylistName(), "important videos");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAvatarUrl() throws Exception {
|
||||
assertTrue(extractor.getAvatarUrl(), extractor.getAvatarUrl().contains("yt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBannerUrl() throws Exception {
|
||||
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderUrl() throws Exception {
|
||||
assertTrue(extractor.getUploaderUrl(), extractor.getUploaderUrl().contains("youtube.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws Exception {
|
||||
assertTrue(extractor.getUploaderName(), !extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderAvatarUrl() throws Exception {
|
||||
assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("yt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsCount() throws Exception {
|
||||
assertTrue("error in the streams count", extractor.getStreamCount() > 100);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
assertTrue("extractor didn't have more streams", extractor.hasMoreStreams());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNextStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
|
||||
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -9,11 +9,12 @@ import org.schabi.newpipe.extractor.search.SearchResult;
|
|||
|
||||
import java.util.EnumSet;
|
||||
|
||||
import static junit.framework.Assert.assertFalse;
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
|
||||
/**
|
||||
/*
|
||||
* Created by Christian Schabesberger on 29.12.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -42,7 +43,7 @@ public class YoutubeSearchEngineAllTest {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
|
||||
SearchEngine engine = Youtube.getService().getSearchEngine();
|
||||
|
||||
// Youtube will suggest "asdf" instead of "asdgff"
|
||||
// keep in mind that the suggestions can change by country (the parameter "de")
|
||||
|
|
|
|||
|
|
@ -10,12 +10,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
|
|||
|
||||
import java.util.EnumSet;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static junit.framework.Assert.assertFalse;
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
|
||||
/**
|
||||
/*
|
||||
* Created by Christian Schabesberger on 29.12.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -44,7 +45,7 @@ public class YoutubeSearchEngineChannelTest {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
|
||||
SearchEngine engine = Youtube.getService().getSearchEngine();
|
||||
|
||||
// Youtube will suggest "gronkh" instead of "grrunkh"
|
||||
// keep in mind that the suggestions can change by country (the parameter "de")
|
||||
|
|
@ -59,7 +60,9 @@ public class YoutubeSearchEngineChannelTest {
|
|||
|
||||
@Test
|
||||
public void testChannelItemType() {
|
||||
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.CHANNEL);
|
||||
for (InfoItem infoItem : result.resultList) {
|
||||
assertEquals(InfoItem.InfoType.CHANNEL, infoItem.info_type);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
|
|||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
|
|
@ -10,12 +11,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
|
|||
|
||||
import java.util.EnumSet;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static junit.framework.Assert.assertFalse;
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
|
||||
/**
|
||||
/*
|
||||
* Created by Christian Schabesberger on 29.12.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -44,7 +46,7 @@ public class YoutubeSearchEngineStreamTest {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
|
||||
SearchEngine engine = Youtube.getService().getSearchEngine();
|
||||
|
||||
// Youtube will suggest "results" instead of "rsults",
|
||||
// keep in mind that the suggestions can change by country (the parameter "de")
|
||||
|
|
@ -58,8 +60,10 @@ public class YoutubeSearchEngineStreamTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testChannelItemType() {
|
||||
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.STREAM);
|
||||
public void testStreamItemType() {
|
||||
for (InfoItem infoItem : result.resultList) {
|
||||
assertEquals(InfoItem.InfoType.STREAM, infoItem.info_type);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -1,21 +1,23 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
||||
/**
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 30.12.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
|
|
@ -45,8 +47,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = NewPipe.getService("Youtube")
|
||||
.getStreamExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A");
|
||||
extractor = Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -56,10 +57,8 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetValidTimeStamp() throws ExtractionException, IOException {
|
||||
StreamExtractor extractor =
|
||||
NewPipe.getService("Youtube")
|
||||
.getStreamExtractorInstance("https://youtu.be/FmG385_uUys?t=174");
|
||||
public void testGetValidTimeStamp() throws IOException, ExtractionException {
|
||||
StreamExtractor extractor = Youtube.getService().getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
|
||||
assertTrue(Integer.toString(extractor.getTimeStamp()),
|
||||
extractor.getTimeStamp() == 174);
|
||||
}
|
||||
|
|
@ -113,12 +112,12 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetAudioStreams() throws ParsingException, ReCaptchaException, IOException {
|
||||
public void testGetAudioStreams() throws IOException, ExtractionException {
|
||||
assertTrue(!extractor.getAudioStreams().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetVideoStreams() throws ParsingException {
|
||||
public void testGetVideoStreams() throws IOException, ExtractionException {
|
||||
for(VideoStream s : extractor.getVideoStreams()) {
|
||||
assertTrue(s.url,
|
||||
s.url.contains(HTTPS));
|
||||
|
|
@ -138,4 +137,11 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
assertTrue(extractor.getDashMpdUrl(),
|
||||
extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRelatedVideos() throws ExtractionException, IOException {
|
||||
StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos();
|
||||
assertFalse(relatedVideos.getItemList().isEmpty());
|
||||
assertTrue(relatedVideos.getErrors().isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 30.12.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeVideoExtractorGema.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This exception is only thrown in Germany.
|
||||
*
|
||||
* WARNING: Deactivate this Test Case before uploading it to Github, otherwise CI will fail.
|
||||
*/
|
||||
@Ignore
|
||||
public class YoutubeStreamExtractorGemaTest {
|
||||
|
||||
@Test
|
||||
public void testGemaError() throws IOException, ExtractionException {
|
||||
try {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
|
||||
|
||||
fail("GemaException should be thrown");
|
||||
} catch (YoutubeStreamExtractor.GemaException ignored) {
|
||||
// Exception was thrown, Gema error detection is working.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeStreamUrlIdHandler}
|
||||
*/
|
||||
public class YoutubeStreamExtractorRestrictedTest {
|
||||
public static final String HTTPS = "https://";
|
||||
private StreamExtractor extractor;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = Youtube.getService()
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvalidTimeStamp() throws ParsingException {
|
||||
assertTrue(Integer.toString(extractor.getTimeStamp()),
|
||||
extractor.getTimeStamp() <= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetValidTimeStamp() throws IOException, ExtractionException {
|
||||
StreamExtractor extractor= Youtube.getService()
|
||||
.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
|
||||
assertTrue(Integer.toString(extractor.getTimeStamp()),
|
||||
extractor.getTimeStamp() == 174);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAgeLimit() throws ParsingException {
|
||||
assertTrue(extractor.getAgeLimit() == 18);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertTrue(!extractor.getTitle().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertTrue(extractor.getDescription() != null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploader() throws ParsingException {
|
||||
assertTrue(!extractor.getUploader().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetViews() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploadDate() throws ParsingException {
|
||||
assertTrue(extractor.getUploadDate().length() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetThumbnailUrl() throws ParsingException {
|
||||
assertTrue(extractor.getThumbnailUrl(),
|
||||
extractor.getThumbnailUrl().contains(HTTPS));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderThumbnailUrl() throws ParsingException {
|
||||
assertTrue(extractor.getUploaderThumbnailUrl(),
|
||||
extractor.getUploaderThumbnailUrl().contains(HTTPS));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAudioStreams() throws IOException, ExtractionException {
|
||||
// audiostream not always necessary
|
||||
assertTrue(!extractor.getAudioStreams().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetVideoStreams() throws IOException, ExtractionException {
|
||||
for(VideoStream s : extractor.getVideoStreams()) {
|
||||
assertTrue(s.url,
|
||||
s.url.contains(HTTPS));
|
||||
assertTrue(s.resolution.length() > 0);
|
||||
assertTrue(Integer.toString(s.format),
|
||||
0 <= s.format && s.format <= 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeStreamUrlIdHandler}
|
||||
*/
|
||||
public class YoutubeStreamUrlIdHandlerTest {
|
||||
private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew";
|
||||
private YoutubeStreamUrlIdHandler urlIdHandler;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
}
|
||||
|
||||
@Test(expected = NullPointerException.class)
|
||||
public void getIdWithNullAsUrl() throws ParsingException {
|
||||
urlIdHandler.getId(null);
|
||||
}
|
||||
|
||||
@Test(expected = FoundAdException.class)
|
||||
public void getIdForAd() throws ParsingException {
|
||||
urlIdHandler.getId(AD_URL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getIdForInvalidUrls() throws ParsingException {
|
||||
List<String> invalidUrls = new ArrayList<>(50);
|
||||
invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
|
||||
invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
|
||||
invalidUrls.add("https://www.youtube.com/");
|
||||
for(String invalidUrl: invalidUrls) {
|
||||
Throwable exception = null;
|
||||
try {
|
||||
urlIdHandler.getId(invalidUrl);
|
||||
} catch (ParsingException e) {
|
||||
exception = e;
|
||||
}
|
||||
if(exception == null) {
|
||||
fail("Expected ParsingException for url: " + invalidUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
@Test
|
||||
public void getId() throws Exception {
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
|
||||
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
|
||||
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
|
||||
assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
|
||||
assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
|
||||
assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
|
||||
|
||||
// Shared links
|
||||
String sharedId = "7JIArTByb3E";
|
||||
String realId = "Q7JsK50NGaA";
|
||||
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
|
||||
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
|
||||
assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAcceptUrl() {
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertTrue(urlIdHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s"));
|
||||
//assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch/jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
|
||||
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
|
||||
|
||||
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
|
||||
|
||||
String sharedId = "8A940MXKFmQ";
|
||||
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
|
||||
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.SuggestionExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 18.11.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeSuggestionExtractorTest.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Test for {@link SuggestionExtractor}
|
||||
*/
|
||||
public class YoutubeSuggestionExtractorTest {
|
||||
private SuggestionExtractor suggestionExtractor;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
suggestionExtractor = Youtube.getService().getSuggestionExtractor();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIfSuggestions() throws IOException, ExtractionException {
|
||||
assertFalse(suggestionExtractor.suggestionList("hello", "de").isEmpty());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue