Refactor and improvements

This commit is contained in:
Mauricio Colli 2017-08-06 17:20:15 -03:00
parent 731102be39
commit ea5b08db4c
61 changed files with 1458 additions and 857 deletions

View File

@ -10,5 +10,8 @@ dependencies {
implementation 'org.mozilla:rhino:1.7.7.1'
testImplementation 'junit:junit:4.12'
sourceCompatibility = 1.7
targetCompatibility = 1.7
}

View File

@ -1,35 +1,73 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.Serializable;
import java.io.IOException;
public abstract class Extractor implements Serializable {
private final int serviceId;
private final String url;
private final UrlIdHandler urlIdHandler;
private final StreamInfoItemCollector previewInfoCollector;
public abstract class Extractor {
/**
* {@link StreamingService} currently related to this extractor.<br/>
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
*/
private final StreamingService service;
public Extractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
this.urlIdHandler = urlIdHandler;
this.serviceId = serviceId;
this.url = url;
this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
/**
* Dirty/original url that was passed in the constructor.
* <p>
* What makes a url "dirty" or not is, for example, the additional parameters
* (not important asin this casethe id):
* <pre>
* https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i> <i><b>&t=4s</b></i>
* </pre>
* But as you can imagine, the time parameter is very important when calling, for example, {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
*/
private final String originalUrl;
/**
* The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
* <p>
* Is lazily-cleaned by calling {@link #getCleanUrl()}
*/
private String cleanUrl;
public Extractor(StreamingService service, String url) throws ExtractionException {
this.service = service;
this.originalUrl = url;
}
public String getUrl() {
return url;
/**
* @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
*/
protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
/**
* Fetch the current page.
*/
public abstract void fetchPage() throws IOException, ExtractionException;
public String getOriginalUrl() {
return originalUrl;
}
public UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
public String getCleanUrl() {
if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
try {
cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
} catch (Exception e) {
cleanUrl = null;
// Fallback to the original url
return originalUrl;
}
return cleanUrl;
}
public StreamingService getService() {
return service;
}
public int getServiceId() {
return serviceId;
}
protected StreamInfoItemCollector getStreamPreviewInfoCollector() {
return previewInfoCollector;
return service.getServiceId();
}
}

View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
public abstract class Info implements Serializable {
@ -15,5 +15,5 @@ public abstract class Info implements Serializable {
public String url;
public String name;
public List<Throwable> errors = new Vector<>();
public List<Throwable> errors = new ArrayList<>();
}

View File

@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
/*
* Created by Christian Schabesberger on 12.02.17.
@ -26,8 +26,8 @@ import java.util.Vector;
*/
public abstract class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>();
private List<Throwable> errors = new Vector<>();
private List<InfoItem> itemList = new ArrayList<>();
private List<Throwable> errors = new ArrayList<>();
private int serviceId = -1;
public InfoItemCollector(int serviceId) {
@ -46,7 +46,7 @@ public abstract class InfoItemCollector {
if (serviceId != otherC.serviceId) {
throw new ExtractionException("Service Id does not equal: "
+ NewPipe.getNameOfService(serviceId)
+ " and " + NewPipe.getNameOfService(otherC.serviceId));
+ " and " + NewPipe.getNameOfService((otherC.serviceId)));
}
errors.addAll(otherC.errors);
itemList.addAll(otherC.itemList);

View File

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
import java.util.List;
/**
* Base class to extractors that have a list (e.g. playlists, channels).
@ -11,16 +12,40 @@ import java.io.IOException;
public abstract class ListExtractor extends Extractor {
protected String nextStreamsUrl;
public ListExtractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
super(urlIdHandler, serviceId, url);
/**
* Get a new ListExtractor with the given nextStreamsUrl set.
* <p>
* The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
* <p>
* You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
* downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
*/
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url);
setNextStreamsUrl(nextStreamsUrl);
if (fetchPageUponCreation()) {
fetchPage();
}
}
/**
* Decide if the page will be fetched upon creation.
* <p>
* The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
* don't need or know what is the next page, thus, fetch the page).
*/
protected boolean fetchPageUponCreation() {
return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
}
public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException;
public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException;
public boolean hasMoreStreams() {
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
}
public abstract StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException;
public String getNextStreamsUrl() {
return nextStreamsUrl;
}
@ -29,4 +54,29 @@ public abstract class ListExtractor extends Extractor {
this.nextStreamsUrl = nextStreamsUrl;
}
/*//////////////////////////////////////////////////////////////////////////
// Inner
//////////////////////////////////////////////////////////////////////////*/
public static class NextItemsResult {
/**
* The current list of items to this result
*/
public final List<InfoItem> nextItemsList;
/**
* Next url to fetch more items
*/
public final String nextItemsUrl;
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
this.nextItemsList = nextItemsList;
this.nextItemsUrl = nextItemsUrl;
}
public boolean hasMoreStreams() {
return nextItemsUrl != null && !nextItemsUrl.isEmpty();
}
}
}

View File

@ -0,0 +1,9 @@
package org.schabi.newpipe.extractor;
import java.util.List;
public abstract class ListInfo extends Info {
public List<InfoItem> related_streams;
public boolean has_more_streams;
public String next_streams_url;
}

View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
/*
* Created by Christian Schabesberger on 23.08.15.
*
@ -22,54 +20,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Provides access to the video streaming services supported by NewPipe.
* Currently only Youtube until the API becomes more stable.
*/
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@SuppressWarnings("ALL")
/**
* Provides access to streaming services supported by NewPipe.
*/
public class NewPipe {
private static final String TAG = NewPipe.class.toString();
private NewPipe() {
}
private static Downloader downloader = null;
public static StreamingService[] getServices() {
return ServiceList.serviceList;
}
public static StreamingService getService(int serviceId) throws ExtractionException {
for (StreamingService s : ServiceList.serviceList) {
if (s.getServiceId() == serviceId) {
return s;
}
}
return null;
}
public static StreamingService getService(String serviceName) throws ExtractionException {
return ServiceList.serviceList[getIdOfService(serviceName)];
}
public static String getNameOfService(int id) {
try {
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "";
}
}
public static int getIdOfService(String serviceName) {
for (int i = 0; i < ServiceList.serviceList.length; i++) {
if (ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
return i;
}
}
return -1;
private NewPipe() {
}
public static void init(Downloader d) {
@ -80,12 +40,63 @@ public class NewPipe {
return downloader;
}
public static StreamingService getServiceByUrl(String url) {
for (StreamingService s : ServiceList.serviceList) {
if (s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
return s;
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
public static StreamingService[] getServices() {
final ServiceList[] values = ServiceList.values();
final StreamingService[] streamingServices = new StreamingService[values.length];
for (int i = 0; i < values.length; i++) streamingServices[i] = values[i].getService();
return streamingServices;
}
public static StreamingService getService(int serviceId) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getServiceId() == serviceId) {
return item.getService();
}
}
return null;
throw new ExtractionException("There's no service with the id = \"" + serviceId + "\"");
}
public static StreamingService getService(String serviceName) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getServiceInfo().name.equals(serviceName)) {
return item.getService();
}
}
throw new ExtractionException("There's no service with the name = \"" + serviceName + "\"");
}
public static StreamingService getServiceByUrl(String url) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
return item.getService();
}
}
throw new ExtractionException("No service can handle the url = \"" + url + "\"");
}
public static int getIdOfService(String serviceName) {
try {
//noinspection ConstantConditions
return getService(serviceName).getServiceId();
} catch (ExtractionException ignored) {
return -1;
}
}
public static String getNameOfService(int id) {
try {
//noinspection ConstantConditions
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "<unknown>";
}
}
}

View File

@ -1,15 +1,36 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/*
* Created by the-scrabi on 18.02.17.
/**
* A list of supported services.
*/
public enum ServiceList {
Youtube(new YoutubeService(0, "Youtube")),
SoundCloud(new SoundcloudService(1, "SoundCloud"));
// DailyMotion(new DailyMotionService(2, "DailyMotion"));
class ServiceList {
public static final StreamingService[] serviceList = {
new YoutubeService(0),
new SoundcloudService(1)
};
private final StreamingService service;
ServiceList(StreamingService service) {
this.service = service;
}
public StreamingService getService() {
return service;
}
public StreamingService.ServiceInfo getServiceInfo() {
return service.getServiceInfo();
}
public int getId() {
return service.getServiceId();
}
@Override
public String toString() {
return service.getServiceInfo().name;
}
}

View File

@ -10,7 +10,11 @@ import java.io.IOException;
public abstract class StreamingService {
public class ServiceInfo {
public String name = "";
public final String name;
public ServiceInfo(String name) {
this.name = name;
}
}
public enum LinkType {
@ -20,35 +24,46 @@ public abstract class StreamingService {
PLAYLIST
}
private int serviceId;
private final int serviceId;
private final ServiceInfo serviceInfo;
public StreamingService(int id) {
serviceId = id;
public StreamingService(int id, String name) {
this.serviceId = id;
this.serviceInfo = new ServiceInfo(name);
}
public abstract ServiceInfo getServiceInfo();
public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
public abstract UrlIdHandler getPlaylistUrlIdHandlerInstance();
public abstract SearchEngine getSearchEngineInstance();
public abstract SuggestionExtractor getSuggestionExtractorInstance();
public abstract StreamExtractor getStreamExtractorInstance(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException;
public abstract PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException;
public final int getServiceId() {
return serviceId;
}
public ServiceInfo getServiceInfo() {
return serviceInfo;
}
public abstract UrlIdHandler getStreamUrlIdHandler();
public abstract UrlIdHandler getChannelUrlIdHandler();
public abstract UrlIdHandler getPlaylistUrlIdHandler();
public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null);
}
public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
return getPlaylistExtractor(url, null);
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/
public final LinkType getLinkTypeByUrl(String url) {
UrlIdHandler sH = getStreamUrlIdHandlerInstance();
UrlIdHandler cH = getChannelUrlIdHandlerInstance();
UrlIdHandler pH = getPlaylistUrlIdHandlerInstance();
UrlIdHandler sH = getStreamUrlIdHandler();
UrlIdHandler cH = getChannelUrlIdHandler();
UrlIdHandler pH = getPlaylistUrlIdHandler();
if (sH.acceptUrl(url)) {
return LinkType.STREAM;

View File

@ -33,8 +33,7 @@ public abstract class SuggestionExtractor {
this.serviceId = serviceId;
}
public abstract List<String> suggestionList(String query, String contentCountry)
throws ExtractionException, IOException;
public abstract List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException;
public int getServiceId() {
return serviceId;

View File

@ -1,9 +1,6 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
/*
* Created by Christian Schabesberger on 26.07.16.

View File

@ -1,11 +1,10 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@ -31,8 +30,13 @@ import java.io.IOException;
public abstract class ChannelExtractor extends ListExtractor {
public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, serviceId, url);
public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getChannelUrlIdHandler();
}
public abstract String getChannelId() throws ParsingException;
@ -40,7 +44,6 @@ public abstract class ChannelExtractor extends ListExtractor {
public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract long getSubscriberCount() throws ParsingException;
}

View File

@ -1,11 +1,16 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List;
import java.io.IOException;
import java.util.ArrayList;
/*
* Created by Christian Schabesberger on 31.07.16.
@ -27,17 +32,36 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfo extends Info {
public class ChannelInfo extends ListInfo {
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
}
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
return service.getChannelExtractor(null, nextStreamsUrl).getNextStreams();
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static ChannelInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getChannelExtractor(url));
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
ChannelInfo info = new ChannelInfo();
// important data
info.service_id = extractor.getServiceId();
info.url = extractor.getUrl();
info.url = extractor.getCleanUrl();
info.id = extractor.getChannelId();
info.name = extractor.getChannelName();
info.has_more_streams = extractor.hasMoreStreams();
try {
info.avatar_url = extractor.getAvatarUrl();
@ -67,13 +91,16 @@ public class ChannelInfo extends Info {
info.errors.add(e);
}
// Lists can be null if a exception was thrown during extraction
if (info.related_streams == null) info.related_streams = new ArrayList<>();
info.has_more_streams = extractor.hasMoreStreams();
info.next_streams_url = extractor.getNextStreamsUrl();
return info;
}
public String avatar_url;
public String banner_url;
public String feed_url;
public List<InfoItem> related_streams;
public long subscriber_count = -1;
public boolean has_more_streams = false;
}

View File

@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem {
public String thumbnail_url;
public String description;
public long subscriber_count = -1;
public long view_count = -1;
public long stream_count = -1;
public ChannelInfoItem() {
super(InfoType.CHANNEL);

View File

@ -43,7 +43,7 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
addError(e);
}
try {
resultItem.view_count = extractor.getViewCount();
resultItem.stream_count = extractor.getStreamCount();
} catch (Exception e) {
addError(e);
}

View File

@ -28,5 +28,5 @@ public interface ChannelInfoItemExtractor {
String getWebPageUrl() throws ParsingException;
String getDescription() throws ParsingException;
long getSubscriberCount() throws ParsingException;
long getViewCount() throws ParsingException;
long getStreamCount() throws ParsingException;
}

View File

@ -1,18 +1,22 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
public abstract class PlaylistExtractor extends ListExtractor {
public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, serviceId, url);
public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getPlaylistUrlIdHandler();
}
public abstract String getPlaylistId() throws ParsingException;
@ -22,6 +26,5 @@ public abstract class PlaylistExtractor extends ListExtractor {
public abstract String getUploaderUrl() throws ParsingException;
public abstract String getUploaderName() throws ParsingException;
public abstract String getUploaderAvatarUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract long getStreamsCount() throws ParsingException;
public abstract long getStreamCount() throws ParsingException;
}

View File

@ -1,25 +1,49 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List;
import java.io.IOException;
import java.util.ArrayList;
public class PlaylistInfo extends Info {
public class PlaylistInfo extends ListInfo {
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
}
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(null, nextStreamsUrl).getNextStreams();
}
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static PlaylistInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getPlaylistExtractor(url));
}
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
PlaylistInfo info = new PlaylistInfo();
info.service_id = extractor.getServiceId();
info.url = extractor.getUrl();
info.url = extractor.getCleanUrl();
info.id = extractor.getPlaylistId();
info.name = extractor.getPlaylistName();
info.has_more_streams = extractor.hasMoreStreams();
try {
info.streams_count = extractor.getStreamsCount();
info.stream_count = extractor.getStreamCount();
} catch (Exception e) {
info.errors.add(e);
}
@ -56,6 +80,11 @@ public class PlaylistInfo extends Info {
info.errors.add(e);
}
// Lists can be null if a exception was thrown during extraction
if (info.related_streams == null) info.related_streams = new ArrayList<>();
info.has_more_streams = extractor.hasMoreStreams();
info.next_streams_url = extractor.getNextStreamsUrl();
return info;
}
@ -64,7 +93,5 @@ public class PlaylistInfo extends Info {
public String uploader_url;
public String uploader_name;
public String uploader_avatar_url;
public long streams_count = 0;
public List<InfoItem> related_streams;
public boolean has_more_streams;
public long stream_count = 0;
}

View File

@ -8,7 +8,7 @@ public class PlaylistInfoItem extends InfoItem {
/**
* How many streams this playlist have
*/
public long streams_count = 0;
public long stream_count = 0;
public PlaylistInfoItem() {
super(InfoType.PLAYLIST);

View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector {
@ -22,7 +21,7 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
addError(e);
}
try {
resultItem.streams_count = extractor.getStreamsCount();
resultItem.stream_count = extractor.getStreamCount();
} catch (Exception e) {
addError(e);
}

View File

@ -6,5 +6,5 @@ public interface PlaylistInfoItemExtractor {
String getThumbnailUrl() throws ParsingException;
String getPlaylistName() throws ParsingException;
String getWebPageUrl() throws ParsingException;
long getStreamsCount() throws ParsingException;
long getStreamCount() throws ParsingException;
}

View File

@ -59,7 +59,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
try {
result.resultList.add(streamCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found add");
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
@ -69,7 +69,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
try {
result.resultList.add(channelCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found add");
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}

View File

@ -49,5 +49,5 @@ public abstract class SearchEngine {
//Result search(String query, int page);
public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter)
throws ExtractionException, IOException;
throws IOException, ExtractionException;
}

View File

@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Vector;
/*
* Created by Christian Schabesberger on 29.02.16.
@ -31,7 +31,7 @@ import java.util.Vector;
public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query,
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
throws ExtractionException, IOException {
throws IOException, ExtractionException {
SearchResult result = engine
.search(query, page, languageCode, filter)
@ -50,6 +50,6 @@ public class SearchResult {
}
public String suggestion;
public List<InfoItem> resultList = new Vector<>();
public List<Throwable> errors = new Vector<>();
public List<InfoItem> resultList = new ArrayList<>();
public List<Throwable> errors = new ArrayList<>();
}

View File

@ -1,37 +1,46 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String channelId;
private JSONObject channel;
private String nextUrl;
public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, url, serviceId);
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
channelId = urlIdHandler.getId(url);
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
channelId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/users/" + channelId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
channel = new JSONObject(response);
}
@Override
public String getCleanUrl() {
try {
return channel.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override
public String getChannelId() {
return channelId;
@ -56,32 +65,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
}
@Override
public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"
+ "&offset=0"
+ "&linked_partitioning=1";
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
}
@Override
public long getSubscriberCount() {
return channel.getLong("followers_count");
@ -93,26 +76,27 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
if (nextUrl.equals("")) {
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getChannelId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Channel doesn't have more streams");
}
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
String response = dl.download(nextUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
}
}

View File

@ -31,7 +31,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
}
@Override
public long getViewCount() {
public long getStreamCount() {
return searchResult.getLong("track_count");
}

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -21,47 +18,28 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
@Override
public String getUrl(String channelId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + channelId);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String siteUrl) throws ParsingException {
public String getId(String url) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(siteUrl);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(19);
return id;
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String cleanUrl(String siteUrl) throws ParsingException {
public String cleanUrl(String complexUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(siteUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
return ogElement.attr("content");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
@ -71,6 +49,5 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
public boolean acceptUrl(String channelUrl) {
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
return Parser.isMatch(regex, channelUrl.toLowerCase());
}
}

View File

@ -1,13 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -15,61 +9,51 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class SoundcloudParsingHelper {
private static String clientId;
private SoundcloudParsingHelper() {
}
public static final String clientId() throws ReCaptchaException, IOException, RegexException {
public static String clientId() throws ReCaptchaException, IOException, RegexException {
if (clientId != null && !clientId.isEmpty()) return clientId;
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://soundcloud.com");
Document doc = Jsoup.parse(response);
// TODO: Find a less heavy way to get the client_id
// Currently we are downloading a 1MB file (!) just to get the client_id,
// youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid.
// The embed mode has a way to get it, but we still have to download a heavy file (~800KB).
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
String js = dl.download(jsElement.attr("src"));
String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
return clientId;
}
public static String toTimeAgoString(String time) throws ParsingException {
try {
List<Long> times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30),
TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1),
TimeUnit.SECONDS.toMillis(1));
List<String> timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second");
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime();
StringBuilder timeAgoString = new StringBuilder();
for (int i = 0; i < times.size(); i++) {
Long current = times.get(i);
long currentAmount = timeAgo / current;
if (currentAmount > 0) {
timeAgoString.append(currentAmount).append(" ").append(timesString.get(i))
.append(currentAmount != 1 ? "s ago" : " ago");
break;
}
}
if (timeAgoString.toString().equals("")) {
timeAgoString.append("Just now");
}
return timeAgoString.toString();
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
}
public static String toDateString(String time) throws ParsingException {
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
Date date = dateFormat.parse(time);
Date date;
// Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
} catch (Exception e) {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
}
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return newDateFormat.format(date);
} catch (ParseException e) {
@ -77,4 +61,83 @@ public class SoundcloudParsingHelper {
}
}
/**
* Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
return new JSONObject(NewPipe.getDownloader().download(apiUrl));
}
/**
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).<br/>
*
* @return the url resolved
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"));
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
/**
* Fetch the embed player with the url and return the id (like the id from the json api).<br/>
*
* @return the id resolved
*/
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"));
return Parser.matchGroup1(",\"id\":(.*?),", response);
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always
* get MIN_ITEMS or more items.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
*/
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl);
}
return nextStreamsUrl;
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
}
String nextStreamsUrl;
try {
nextStreamsUrl = responseObject.getString("next_href");
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
nextStreamsUrl = "";
}
return nextStreamsUrl;
}
}

View File

@ -1,38 +1,46 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JSONObject playlist;
private List<String> nextTracks;
public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
super(urlIdHandler, url, serviceId);
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
playlistId = urlIdHandler.getId(url);
String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
playlistId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
String response = dl.download(apiUrl);
playlist = new JSONObject(response);
}
@Override
public String getCleanUrl() {
try {
return playlist.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override
public String getPlaylistId() {
return playlistId;
@ -69,61 +77,33 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
@Override
public long getStreamsCount() {
public long getStreamCount() {
return playlist.getLong("track_count");
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
// Note the "api", NOT "api-v2"
String apiUrl = "https://api.soundcloud.com/playlists/" + getPlaylistId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
try {
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
} catch (Exception e) {
nextTracks.add(track.getString("id"));
}
}
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
}
@Override
public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException {
if (nextTracks.equals(null)) {
return null;
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
// TODO: Do this per 10 tracks, instead of all tracks at once
String apiUrl = "https://api-v2.soundcloud.com/tracks?ids=";
for (String id : nextTracks) {
apiUrl += id;
if (!id.equals(nextTracks.get(nextTracks.size() - 1))) {
apiUrl += ",";
}
}
apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
nextTracks = null;
return collector;
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
}
}

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -21,13 +18,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override
public String getUrl(String listId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + listId);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
@ -36,15 +27,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override
public String getId(String url) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(23);
return id;
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
@ -53,15 +36,10 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(complexUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
return ogElement.attr("content");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

View File

@ -1,9 +1,5 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8";

View File

@ -13,61 +13,48 @@ import java.io.IOException;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
super(id);
public SoundcloudService(int id, String name) {
super(id, name);
}
@Override
public ServiceInfo getServiceInfo() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Soundcloud";
return serviceInfo;
}
@Override
public StreamExtractor getStreamExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance();
if (urlIdHandler.acceptUrl(url)) {
return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId());
} else {
throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
public SearchEngine getSearchEngine() {
return new SoundcloudSearchEngine(getServiceId());
}
@Override
public UrlIdHandler getStreamUrlIdHandlerInstance() {
public UrlIdHandler getStreamUrlIdHandler() {
return SoundcloudStreamUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getChannelUrlIdHandlerInstance() {
public UrlIdHandler getChannelUrlIdHandler() {
return SoundcloudChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
public UrlIdHandler getPlaylistUrlIdHandler() {
return SoundcloudPlaylistUrlIdHandler.getInstance();
}
@Override
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
return new SoundcloudStreamExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new SoundcloudChannelExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractorInstance() {
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new SoundcloudSuggestionExtractor(getServiceId());
}
}

View File

@ -1,19 +1,14 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@ -21,33 +16,39 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor {
private String pageUrl;
private String trackId;
private JSONObject track;
public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
}
Downloader dl = NewPipe.getDownloader();
trackId = urlIdHandler.getId(pageUrl);
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
track = new JSONObject(response);
@Override
public void fetchPage() throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
}
}
@Override
public String getCleanUrl() {
try {
return track.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override
public String getId() {
return trackId;
return track.getInt("id") + "";
}
@Override
@ -96,11 +97,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Override
public List<AudioStream> getAudioStreams() throws ReCaptchaException, IOException, RegexException {
Vector<AudioStream> audioStreams = new Vector<>();
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
List<AudioStream> audioStreams = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams"
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + getId() + "/streams"
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
@ -113,12 +114,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoStreams() {
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
return null;
}
@Override
public List<VideoStream> getVideoOnlyStreams() {
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
return null;
}
@ -126,7 +127,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
@ -190,16 +191,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Override
public StreamInfoItemExtractor getNextVideo() {
public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
return null;
}
@Override
public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related"
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related"
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);

View File

@ -35,7 +35,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at"));
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
}
@Override

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -13,6 +10,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private SoundcloudStreamUrlIdHandler() {
}
@ -23,13 +21,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override
public String getUrl(String videoId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + videoId);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
@ -38,15 +30,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override
public String getId(String url) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(20);
return id;
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
@ -55,15 +39,10 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(complexUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
return ogElement.attr("content");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

View File

@ -1,10 +1,5 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
@ -13,6 +8,11 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";

View File

@ -43,7 +43,7 @@ public class ItagItem {
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, AUDIO, WEBMA, 50),
// new ItagItem(250, AUDIO, WEBMA, 70),
// new ItagItem(251, AUDIO, WEBMA, 16),
// new ItagItem(251, AUDIO, WEBMA, 160),
new ItagItem(171, AUDIO, WEBMA, 128),
new ItagItem(172, AUDIO, WEBMA, 256),
new ItagItem(139, AUDIO, M4A, 48),

View File

@ -8,7 +8,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -44,6 +44,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc;
/**
@ -51,29 +52,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/
private Document nextStreamsAjax;
/*//////////////////////////////////////////////////////////////////////////
// Variables for cache purposes (not "select" the current document all over again)
//////////////////////////////////////////////////////////////////////////*/
private String channelId;
private String channelName;
private String avatarUrl;
private String bannerUrl;
private String feedUrl;
private long subscriberCount = -1;
public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
fetchDocument();
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
String userUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextStreamsUrl = getNextStreamsUrlFrom(doc);
nextStreamsAjax = null;
}
@Override
public String getChannelId() throws ParsingException {
try {
if (channelId == null) {
channelId = getUrlIdHandler().getId(getUrl());
}
return channelId;
return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) {
throw new ParsingException("Could not get channel id");
}
@ -82,11 +80,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getChannelName() throws ParsingException {
try {
if (channelName == null) {
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return channelName;
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get channel name");
}
@ -95,11 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
if (avatarUrl == null) {
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
}
return avatarUrl;
return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
} catch (Exception e) {
throw new ParsingException("Could not get avatar", e);
}
@ -108,59 +98,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
if (bannerUrl == null) {
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
}
return bannerUrl;
return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul);
return collector;
}
@Override
public long getSubscriberCount() throws ParsingException {
if (subscriberCount == -1) {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (el != null) {
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
} else {
throw new ParsingException("Could not get subscriber count");
}
}
return subscriberCount;
}
@Override
public String getFeedUrl() throws ParsingException {
try {
if (feedUrl == null) {
String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
}
return feedUrl;
return channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
} catch (Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul);
return collector;
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Channel doesn't have more streams");
}
@ -169,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return collector;
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@ -182,8 +160,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
} else {
nextStreamsUrl = "";
}
@ -192,7 +169,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
private String getNextStreamsUrl(Document d) throws ParsingException {
private String getNextStreamsUrlFrom(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) {
@ -206,17 +183,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000";
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
@ -230,7 +196,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
}
@Override
@ -259,7 +226,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("span[class=\"video-time\"]").first().text());
li.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) {
if (isLiveStream(li)) {
// -1 for no duration

View File

@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
}
@Override
public long getViewCount() throws ParsingException {
public long getStreamCount() throws ParsingException {
Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
if (metaEl == null) {
return 0;

View File

@ -7,6 +7,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -23,39 +24,31 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc = null;
private Document doc;
/**
* It's lazily initialized (when getNextStreams is called)
*/
private Document nextStreamsAjax = null;
private Document nextStreamsAjax;
/*//////////////////////////////////////////////////////////////////////////
// Variables for cache purposes (not "select" the current document all over again)
//////////////////////////////////////////////////////////////////////////*/
private String playlistId;
private String playlistName;
private String avatarUrl;
private String bannerUrl;
public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
}
private long streamsCount;
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
private String uploaderUrl;
private String uploaderName;
private String uploaderAvatarUrl;
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
fetchDocument();
nextStreamsUrl = getNextStreamsUrlFrom(doc);
nextStreamsAjax = null;
}
@Override
public String getPlaylistId() throws ParsingException {
try {
if (playlistId == null) {
playlistId = getUrlIdHandler().getId(getUrl());
}
return playlistId;
return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) {
throw new ParsingException("Could not get playlist id");
}
@ -64,11 +57,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getPlaylistName() throws ParsingException {
try {
if (playlistName == null) {
playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
}
return playlistName;
return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get playlist name");
}
@ -77,11 +66,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
if (avatarUrl == null) {
avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
}
return avatarUrl;
return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
} catch (Exception e) {
throw new ParsingException("Could not get playlist avatar");
}
@ -90,18 +75,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
if (bannerUrl == null) {
Element el = doc.select("div[id=\"gh-banner\"] style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
if (url.contains("s.ytimg.com")) {
bannerUrl = null;
return null;
} else {
bannerUrl = url.substring(0, url.indexOf(");"));
}
return url.substring(0, url.indexOf(");"));
}
return bannerUrl;
} catch (Exception e) {
throw new ParsingException("Could not get playlist Banner");
}
@ -110,11 +93,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
try {
if (uploaderUrl == null) {
uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
}
return uploaderUrl;
return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
}
@ -123,11 +102,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderName() throws ParsingException {
try {
if (uploaderName == null) {
uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return uploaderName;
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
}
@ -136,19 +111,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
if (uploaderAvatarUrl == null) {
uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
}
return uploaderAvatarUrl;
return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar");
}
}
@Override
public long getStreamsCount() throws ParsingException {
if (streamsCount <= 0) {
public long getStreamCount() throws ParsingException {
String input;
try {
@ -158,31 +128,28 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
try {
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
return Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string
if (!input.isEmpty()) {
streamsCount = 0;
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
}
return streamsCount;
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
collectStreamsFrom(collector, tbody);
return collector;
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
@ -191,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return collector;
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@ -204,8 +171,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
} else {
nextStreamsUrl = "";
}
@ -214,17 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private String getNextStreamsUrl(Document d) throws ParsingException {
private String getNextStreamsUrlFrom(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) {
@ -241,7 +197,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
for (final Element li : element.children()) {
collector.commit(new StreamInfoItemExtractor() {
@Override
@ -252,7 +208,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getWebPageUrl() throws ParsingException {
try {
return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id"));
return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
@ -300,7 +256,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}

View File

@ -34,61 +34,48 @@ import java.io.IOException;
public class YoutubeService extends StreamingService {
public YoutubeService(int id) {
super(id);
public YoutubeService(int id, String name) {
super(id, name);
}
@Override
public ServiceInfo getServiceInfo() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Youtube";
return serviceInfo;
}
@Override
public StreamExtractor getStreamExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
if (urlIdHandler.acceptUrl(url)) {
return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
} else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
public SearchEngine getSearchEngine() {
return new YoutubeSearchEngine(getServiceId());
}
@Override
public UrlIdHandler getStreamUrlIdHandlerInstance() {
public UrlIdHandler getStreamUrlIdHandler() {
return YoutubeStreamUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getChannelUrlIdHandlerInstance() {
public UrlIdHandler getChannelUrlIdHandler() {
return YoutubeChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
public UrlIdHandler getPlaylistUrlIdHandler() {
return YoutubePlaylistUrlIdHandler.getInstance();
}
@Override
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
return new YoutubeStreamExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new YoutubeChannelExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractorInstance() {
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new YoutubeSuggestionExtractor(getServiceId());
}
}

View File

@ -10,7 +10,7 @@ import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -26,9 +26,9 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -80,12 +80,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
private Document doc;
private final String dirtyUrl;
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
dirtyUrl = pageUrl;
fetchDocument();
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
}
/*//////////////////////////////////////////////////////////////////////////
@ -95,7 +92,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getId() throws ParsingException {
try {
return getUrlIdHandler().getId(getUrl());
return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) {
throw new ParsingException("Could not get stream id");
}
@ -238,8 +235,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>();
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
List<AudioStream> audioStreams = new ArrayList<>();
try {
String encodedUrlMap;
// playerArgs could be null if the video is age restricted
@ -288,8 +285,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoStream> videoStreams = new Vector<>();
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
List<VideoStream> videoStreams = new ArrayList<>();
try {
String encodedUrlMap;
@ -342,8 +339,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws ParsingException {
Vector<VideoStream> videoOnlyStreams = new Vector<>();
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
List<VideoStream> videoOnlyStreams = new ArrayList<>();
try {
String encodedUrlMap;
@ -405,7 +402,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
@ -516,7 +513,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public StreamInfoItemExtractor getNextVideo() throws ParsingException {
public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
@ -526,9 +523,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public StreamInfoItemCollector getRelatedVideos() throws ParsingException {
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
try {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[id=\"watch-related\"]").first();
if (ul != null) {
for (Element li : ul.children()) {
@ -617,11 +614,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// cached values
private static volatile String decryptionCode = "";
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
JSONObject ytPlayerConfig;
String playerUrl;
@ -632,7 +630,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
playerUrl = getPlayerUrlFromRestrictedVideo();
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
@ -683,7 +681,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
throw new LiveStreamException("This is a Live stream. Can't use those right now.");
}
return playerArgs;
@ -709,12 +707,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
private String getPlayerUrlFromRestrictedVideo() throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = getUrlIdHandler().getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedUrl = "https://www.youtube.com/embed/" + getId();
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
@ -811,7 +808,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
}
@Override
@ -829,8 +827,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
li.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) {
if (isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
}
@Override
@ -845,12 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text()));
@ -875,6 +876,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
return thumbnailUrl;
}
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if (bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if (item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
};
}
}

View File

@ -59,7 +59,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
item.select("span[class=\"video-time\"]").first().text());
item.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) {
if (isLiveStream(item)) {
// -1 for no duration
@ -104,16 +104,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (div == null) {
return -1;
} else {
input = div.select("li").get(1)
.text();
input = div.select("li").get(1).text();
}
} catch (IndexOutOfBoundsException e) {
if (isLiveStream(item)) {
// -1 for no view count
return -1;
} else {
throw new ParsingException(
"Could not parse yt-lockup-meta although available: " + getTitle(), e);
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getTitle(), e);
}
}
@ -161,7 +159,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public boolean isAd() throws ParsingException {
return !item.select("span[class*=\"icon-not-available\"]").isEmpty();
return !item.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!item.select("span[class*=\"yt-badge-ad\"]").isEmpty();
}
private boolean isLiveStream(Element item) {

View File

@ -1,25 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONArray;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/*
* Created by Christian Schabesberger on 28.09.16.
*
@ -49,51 +41,24 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
}
@Override
public List<String> suggestionList(
String query, String contentCountry)
throws ExtractionException, IOException {
public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String url = "https://suggestqueries.google.com/complete/search"
+ "?client=" + ""
+ "&output=" + "toolbar"
+ "?client=" + "firefox" // 'toolbar' for xml
+ "&ds=" + "yt"
+ "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(
new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
throw new ParsingException("Could not parse document.");
}
try {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
return suggestions;
JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1);
for (Object suggestion : suggestionsArray) suggestions.add(suggestion.toString());
} catch (Exception e) {
throw new ParsingException("Could not get suggestions form document.", e);
}
throw new ParsingException("Could not parse suggestions response.", e);
}
return suggestions;
}
}

View File

@ -13,7 +13,7 @@ public abstract class Stream implements Serializable {
}
/**
* Reveals whether two streams are the same, but have different urls
* Reveals whether two streams have the same stats (format and bitrate, for example)
*/
public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format;

View File

@ -21,9 +21,10 @@ package org.schabi.newpipe.extractor.stream;
*/
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.util.List;
@ -33,8 +34,14 @@ import java.util.List;
*/
public abstract class StreamExtractor extends Extractor {
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
super(urlIdHandler, serviceId, url);
public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
fetchPage();
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getStreamUrlIdHandler();
}
public abstract String getId() throws ParsingException;
@ -48,16 +55,16 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract List<VideoStream> getVideoStreams() throws ParsingException;
public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException;
public abstract StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException;
public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException;
public abstract StreamType getStreamType() throws ParsingException;
/**

View File

@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
/*
* Created by Christian Schabesberger on 26.08.15.
@ -46,11 +50,23 @@ public class StreamInfo extends Info {
public StreamInfo() {
}
public static StreamInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static StreamInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getStreamExtractor(url));
}
/**
* Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses
*/
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo = new StreamInfo();
try {
@ -80,7 +96,7 @@ public class StreamInfo extends Info {
// if one of these is not available an exception is meant to be thrown directly into the frontend.
streamInfo.service_id = extractor.getServiceId();
streamInfo.url = extractor.getUrl();
streamInfo.url = extractor.getCleanUrl();
streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle();
@ -128,15 +144,12 @@ public class StreamInfo extends Info {
}
// Lists can be null if a exception was thrown during extraction
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
if (streamInfo.video_streams == null) streamInfo.video_streams = new ArrayList<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>();
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try {
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
// it has video, video only and audio streams and will only add to the list if it don't
// find a similar stream in the respective lists (calling Stream#equalStats).
DashMpdParser.getStreams(streamInfo);
} catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the manifest,
@ -246,6 +259,8 @@ public class StreamInfo extends Info {
streamInfo.addException(e);
}
if (streamInfo.related_streams == null) streamInfo.related_streams = new ArrayList<>();
return streamInfo;
}
@ -278,7 +293,7 @@ public class StreamInfo extends Info {
public int dislike_count = -1;
public String average_rating;
public StreamInfoItem next_video;
public List<InfoItem> related_streams = new Vector<>();
public List<InfoItem> related_streams;
//in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0;
}

View File

@ -53,7 +53,13 @@ public class DashMpdParser {
}
/**
* Download manifest and return nodelist with elements of tag "AdaptationSet"
* Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
* then it will search for any stream that the ItagItem has (by the id).
* <p>
* It has video, video only and audio streams and will only add to the list if it don't
* find a similar stream in the respective lists (calling {@link Stream#equalStats}).
*
* @param streamInfo where the parsed streams will be added
*/
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
String dashDoc;

View File

@ -21,17 +21,25 @@ public class Utils {
/**
* Check if throwable have the cause
*/
public static boolean hasCauseThrowable(Throwable throwable, Class<?> causeToCheck) {
public static boolean hasCauseThrowable(Throwable throwable, Class<?>... causesToCheck) {
// Check if getCause is not the same as cause (the getCause is already the root),
// as it will cause a infinite loop if it is
Throwable cause, getCause = throwable;
for (Class<?> causesEl : causesToCheck) {
if (throwable.getClass().isAssignableFrom(causesEl)) {
return true;
}
}
while ((cause = throwable.getCause()) != null && getCause != cause) {
getCause = cause;
if (cause.getClass().isAssignableFrom(causeToCheck)) {
for (Class<?> causesEl : causesToCheck) {
if (cause.getClass().isAssignableFrom(causesEl)) {
return true;
}
}
}
return false;
}
}

View File

@ -14,7 +14,7 @@ import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
/**
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -41,7 +41,8 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static Downloader instance = null;
private Downloader() {}
private Downloader() {
}
public static Downloader getInstance() {
if (instance == null) {
@ -62,11 +63,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return Downloader.mCookies;
}
/**Download the text file at the supplied URL as in download(String),
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
* @return the contents of the specified text file
*/
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
@ -74,12 +78,15 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
/**Download the text file at the supplied URL as in download(String),
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
@ -91,12 +98,15 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
/**
* Common functionality between download(String url) and download(String url, String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
@ -123,7 +133,8 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(e);
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if (in != null) {
in.close();
@ -133,10 +144,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
/**
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();

View File

@ -0,0 +1,74 @@
package org.schabi.newpipe.extractor;
import org.junit.Test;
import java.util.HashSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
public class NewPipeTest {
@Test
public void getAllServicesTest() throws Exception {
assertEquals(NewPipe.getServices().length, ServiceList.values().length);
}
@Test
public void testAllServicesHaveDifferentId() throws Exception {
HashSet<Integer> servicesId = new HashSet<>();
for (StreamingService streamingService : NewPipe.getServices()) {
String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().name + ")";
assertTrue(errorMsg, servicesId.add(streamingService.getServiceId()));
}
}
@Test
public void getServiceWithId() throws Exception {
assertEquals(NewPipe.getService(Youtube.getId()), Youtube.getService());
assertEquals(NewPipe.getService(SoundCloud.getId()), SoundCloud.getService());
assertNotEquals(NewPipe.getService(SoundCloud.getId()), Youtube.getService());
}
@Test
public void getServiceWithName() throws Exception {
assertEquals(NewPipe.getService(Youtube.getServiceInfo().name), Youtube.getService());
assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().name), SoundCloud.getService());
assertNotEquals(NewPipe.getService(Youtube.getServiceInfo().name), SoundCloud.getService());
}
@Test
public void getServiceWithUrl() throws Exception {
assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), Youtube.getService());
assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), Youtube.getService());
assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), Youtube.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud.getService());
assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), Youtube.getService());
assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud.getService());
}
@Test
public void getIdWithServiceName() throws Exception {
assertEquals(NewPipe.getIdOfService(Youtube.getServiceInfo().name), Youtube.getId());
assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), SoundCloud.getId());
assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), Youtube.getId());
}
@Test
public void getServiceNameWithId() throws Exception {
assertEquals(NewPipe.getNameOfService(Youtube.getId()), Youtube.getServiceInfo().name);
assertEquals(NewPipe.getNameOfService(SoundCloud.getId()), SoundCloud.getServiceInfo().name);
assertNotEquals(NewPipe.getNameOfService(Youtube.getId()), SoundCloud.getServiceInfo().name);
}
}

View File

@ -1,16 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertTrue;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
/**
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 12.09.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -41,8 +42,8 @@ public class YoutubeChannelExtractorTest {
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
extractor = Youtube.getService()
.getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
}
@Test
@ -61,7 +62,7 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testGetBannerurl() throws Exception {
public void testGetBannerUrl() throws Exception {
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3"));
}
@ -81,9 +82,10 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testHasNextPage() throws Exception {
// this particular example (link) has a next page !!!
assertTrue("no next page link found", extractor.hasMoreStreams());
public void testHasMoreStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("don't have more streams", extractor.hasMoreStreams());
}
@Test
@ -92,16 +94,11 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testGetNextPage() throws Exception {
extractor = NewPipe.getService("Youtube")
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
assertTrue("next page didn't have content", !extractor.getStreams().getItemList().isEmpty());
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
@Test
public void testGetNextNextPageUrl() throws Exception {
extractor = NewPipe.getService("Youtube")
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
assertTrue("next page didn't have content", extractor.hasMoreStreams());
}
}

View File

@ -0,0 +1,98 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
* Test for {@link PlaylistExtractor}
*/
public class YoutubePlaylistExtractorTest {
private PlaylistExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = Youtube.getService()
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
}
@Test
public void testGetDownloader() throws Exception {
assertNotNull(NewPipe.getDownloader());
}
@Test
public void testGetId() throws Exception {
assertEquals(extractor.getPlaylistId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
}
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getPlaylistName(), "important videos");
}
@Test
public void testGetAvatarUrl() throws Exception {
assertTrue(extractor.getAvatarUrl(), extractor.getAvatarUrl().contains("yt"));
}
@Test
public void testGetBannerUrl() throws Exception {
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt"));
}
@Test
public void testGetUploaderUrl() throws Exception {
assertTrue(extractor.getUploaderUrl(), extractor.getUploaderUrl().contains("youtube.com"));
}
@Test
public void testGetUploaderName() throws Exception {
assertTrue(extractor.getUploaderName(), !extractor.getUploaderName().isEmpty());
}
@Test
public void testGetUploaderAvatarUrl() throws Exception {
assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("yt"));
}
@Test
public void testGetStreamsCount() throws Exception {
assertTrue("error in the streams count", extractor.getStreamCount() > 100);
}
@Test
public void testGetStreams() throws Exception {
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
}
@Test
public void testGetStreamsErrors() throws Exception {
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
}
@Test
public void testHasMoreStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have more streams", extractor.hasMoreStreams());
}
@Test
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
}

View File

@ -9,11 +9,12 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -42,7 +43,7 @@ public class YoutubeSearchEngineAllTest {
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "asdf" instead of "asdgff"
// keep in mind that the suggestions can change by country (the parameter "de")

View File

@ -10,12 +10,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -44,7 +45,7 @@ public class YoutubeSearchEngineChannelTest {
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de")
@ -59,7 +60,9 @@ public class YoutubeSearchEngineChannelTest {
@Test
public void testChannelItemType() {
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.CHANNEL);
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.CHANNEL, infoItem.info_type);
}
}
@Test

View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,12 +11,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -44,7 +46,7 @@ public class YoutubeSearchEngineStreamTest {
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "results" instead of "rsults",
// keep in mind that the suggestions can change by country (the parameter "de")
@ -58,8 +60,10 @@ public class YoutubeSearchEngineStreamTest {
}
@Test
public void testChannelItemType() {
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.STREAM);
public void testStreamItemType() {
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.STREAM, infoItem.info_type);
}
}
@Test

View File

@ -1,21 +1,23 @@
package org.schabi.newpipe.extractor.services.youtube;
import static junit.framework.Assert.assertTrue;
import java.io.IOException;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
/**
import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 30.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -45,8 +47,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getStreamExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A");
extractor = Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
}
@Test
@ -56,10 +57,8 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetValidTimeStamp() throws ExtractionException, IOException {
StreamExtractor extractor =
NewPipe.getService("Youtube")
.getStreamExtractorInstance("https://youtu.be/FmG385_uUys?t=174");
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = Youtube.getService().getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
@ -113,12 +112,12 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetAudioStreams() throws ParsingException, ReCaptchaException, IOException {
public void testGetAudioStreams() throws IOException, ExtractionException {
assertTrue(!extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws ParsingException {
public void testGetVideoStreams() throws IOException, ExtractionException {
for(VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains(HTTPS));
@ -138,4 +137,11 @@ public class YoutubeStreamExtractorDefaultTest {
assertTrue(extractor.getDashMpdUrl(),
extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty());
}
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos();
assertFalse(relatedVideos.getItemList().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
}

View File

@ -0,0 +1,53 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 30.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeVideoExtractorGema.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* This exception is only thrown in Germany.
*
* WARNING: Deactivate this Test Case before uploading it to Github, otherwise CI will fail.
*/
@Ignore
public class YoutubeStreamExtractorGemaTest {
@Test
public void testGemaError() throws IOException, ExtractionException {
try {
NewPipe.init(Downloader.getInstance());
Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
fail("GemaException should be thrown");
} catch (YoutubeStreamExtractor.GemaException ignored) {
// Exception was thrown, Gema error detection is working.
}
}
}

View File

@ -0,0 +1,108 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
* Test for {@link YoutubeStreamUrlIdHandler}
*/
public class YoutubeStreamExtractorRestrictedTest {
public static final String HTTPS = "https://";
private StreamExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = Youtube.getService()
.getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor= Youtube.getService()
.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
@Test
public void testGetAgeLimit() throws ParsingException {
assertTrue(extractor.getAgeLimit() == 18);
}
@Test
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
@Test
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains(HTTPS));
}
@Test
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains(HTTPS));
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audiostream not always necessary
assertTrue(!extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
for(VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains(HTTPS));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.format),
0 <= s.format && s.format <= 4);
}
}
}

View File

@ -0,0 +1,117 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertTrue;
/**
* Test for {@link YoutubeStreamUrlIdHandler}
*/
public class YoutubeStreamUrlIdHandlerTest {
private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew";
private YoutubeStreamUrlIdHandler urlIdHandler;
@Before
public void setUp() throws Exception {
urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
NewPipe.init(Downloader.getInstance());
}
@Test(expected = NullPointerException.class)
public void getIdWithNullAsUrl() throws ParsingException {
urlIdHandler.getId(null);
}
@Test(expected = FoundAdException.class)
public void getIdForAd() throws ParsingException {
urlIdHandler.getId(AD_URL);
}
@Test
public void getIdForInvalidUrls() throws ParsingException {
List<String> invalidUrls = new ArrayList<>(50);
invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
invalidUrls.add("https://www.youtube.com/");
for(String invalidUrl: invalidUrls) {
Throwable exception = null;
try {
urlIdHandler.getId(invalidUrl);
} catch (ParsingException e) {
exception = e;
}
if(exception == null) {
fail("Expected ParsingException for url: " + invalidUrl);
}
}
}
@Test
public void getId() throws Exception {
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
// Shared links
String sharedId = "7JIArTByb3E";
String realId = "Q7JsK50NGaA";
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
}
@Test
public void testAcceptUrl() {
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s"));
//assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
String sharedId = "8A940MXKFmQ";
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId));
}
}

View File

@ -0,0 +1,51 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 18.11.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeSuggestionExtractorTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SuggestionExtractor}
*/
public class YoutubeSuggestionExtractorTest {
private SuggestionExtractor suggestionExtractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
suggestionExtractor = Youtube.getService().getSuggestionExtractor();
}
@Test
public void testIfSuggestions() throws IOException, ExtractionException {
assertFalse(suggestionExtractor.suggestionList("hello", "de").isEmpty());
}
}