Merge pull request #19 from mauriciocolli/refactor

Refactor and improvements
This commit is contained in:
Mauricio Colli 2017-08-06 18:07:39 -03:00 committed by GitHub
commit cfc2b9b3e2
61 changed files with 1458 additions and 857 deletions

View File

@ -10,5 +10,8 @@ dependencies {
implementation 'org.mozilla:rhino:1.7.7.1' implementation 'org.mozilla:rhino:1.7.7.1'
testImplementation 'junit:junit:4.12' testImplementation 'junit:junit:4.12'
sourceCompatibility = 1.7
targetCompatibility = 1.7
} }

View File

@ -1,35 +1,73 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.Serializable; import java.io.IOException;
public abstract class Extractor implements Serializable { public abstract class Extractor {
private final int serviceId; /**
private final String url; * {@link StreamingService} currently related to this extractor.<br/>
private final UrlIdHandler urlIdHandler; * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
private final StreamInfoItemCollector previewInfoCollector; */
private final StreamingService service;
public Extractor(UrlIdHandler urlIdHandler, int serviceId, String url) { /**
this.urlIdHandler = urlIdHandler; * Dirty/original url that was passed in the constructor.
this.serviceId = serviceId; * <p>
this.url = url; * What makes a url "dirty" or not is, for example, the additional parameters
this.previewInfoCollector = new StreamInfoItemCollector(serviceId); * (not important asin this casethe id):
* <pre>
* https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i> <i><b>&t=4s</b></i>
* </pre>
* But as you can imagine, the time parameter is very important when calling, for example, {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
*/
private final String originalUrl;
/**
* The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
* <p>
* Is lazily-cleaned by calling {@link #getCleanUrl()}
*/
private String cleanUrl;
public Extractor(StreamingService service, String url) throws ExtractionException {
this.service = service;
this.originalUrl = url;
} }
public String getUrl() { /**
return url; * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
*/
protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
/**
* Fetch the current page.
*/
public abstract void fetchPage() throws IOException, ExtractionException;
public String getOriginalUrl() {
return originalUrl;
} }
public UrlIdHandler getUrlIdHandler() { public String getCleanUrl() {
return urlIdHandler; if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
try {
cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
} catch (Exception e) {
cleanUrl = null;
// Fallback to the original url
return originalUrl;
}
return cleanUrl;
}
public StreamingService getService() {
return service;
} }
public int getServiceId() { public int getServiceId() {
return serviceId; return service.getServiceId();
}
protected StreamInfoItemCollector getStreamPreviewInfoCollector() {
return previewInfoCollector;
} }
} }

View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Vector;
public abstract class Info implements Serializable { public abstract class Info implements Serializable {
@ -15,5 +15,5 @@ public abstract class Info implements Serializable {
public String url; public String url;
public String name; public String name;
public List<Throwable> errors = new Vector<>(); public List<Throwable> errors = new ArrayList<>();
} }

View File

@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Vector;
/* /*
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
@ -26,8 +26,8 @@ import java.util.Vector;
*/ */
public abstract class InfoItemCollector { public abstract class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>(); private List<InfoItem> itemList = new ArrayList<>();
private List<Throwable> errors = new Vector<>(); private List<Throwable> errors = new ArrayList<>();
private int serviceId = -1; private int serviceId = -1;
public InfoItemCollector(int serviceId) { public InfoItemCollector(int serviceId) {
@ -46,7 +46,7 @@ public abstract class InfoItemCollector {
if (serviceId != otherC.serviceId) { if (serviceId != otherC.serviceId) {
throw new ExtractionException("Service Id does not equal: " throw new ExtractionException("Service Id does not equal: "
+ NewPipe.getNameOfService(serviceId) + NewPipe.getNameOfService(serviceId)
+ " and " + NewPipe.getNameOfService(otherC.serviceId)); + " and " + NewPipe.getNameOfService((otherC.serviceId)));
} }
errors.addAll(otherC.errors); errors.addAll(otherC.errors);
itemList.addAll(otherC.itemList); itemList.addAll(otherC.itemList);

View File

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException; import java.io.IOException;
import java.util.List;
/** /**
* Base class to extractors that have a list (e.g. playlists, channels). * Base class to extractors that have a list (e.g. playlists, channels).
@ -11,16 +12,40 @@ import java.io.IOException;
public abstract class ListExtractor extends Extractor { public abstract class ListExtractor extends Extractor {
protected String nextStreamsUrl; protected String nextStreamsUrl;
public ListExtractor(UrlIdHandler urlIdHandler, int serviceId, String url) { /**
super(urlIdHandler, serviceId, url); * Get a new ListExtractor with the given nextStreamsUrl set.
* <p>
* The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
* <p>
* You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
* downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
*/
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url);
setNextStreamsUrl(nextStreamsUrl);
if (fetchPageUponCreation()) {
fetchPage();
}
} }
public boolean hasMoreStreams(){ /**
* Decide if the page will be fetched upon creation.
* <p>
* The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
* don't need or know what is the next page, thus, fetch the page).
*/
protected boolean fetchPageUponCreation() {
return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
}
public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException;
public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException;
public boolean hasMoreStreams() {
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty(); return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
} }
public abstract StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException;
public String getNextStreamsUrl() { public String getNextStreamsUrl() {
return nextStreamsUrl; return nextStreamsUrl;
} }
@ -29,4 +54,29 @@ public abstract class ListExtractor extends Extractor {
this.nextStreamsUrl = nextStreamsUrl; this.nextStreamsUrl = nextStreamsUrl;
} }
/*//////////////////////////////////////////////////////////////////////////
// Inner
//////////////////////////////////////////////////////////////////////////*/
public static class NextItemsResult {
/**
* The current list of items to this result
*/
public final List<InfoItem> nextItemsList;
/**
* Next url to fetch more items
*/
public final String nextItemsUrl;
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
this.nextItemsList = nextItemsList;
this.nextItemsUrl = nextItemsUrl;
}
public boolean hasMoreStreams() {
return nextItemsUrl != null && !nextItemsUrl.isEmpty();
}
}
} }

View File

@ -0,0 +1,9 @@
package org.schabi.newpipe.extractor;
import java.util.List;
public abstract class ListInfo extends Info {
public List<InfoItem> related_streams;
public boolean has_more_streams;
public String next_streams_url;
}

View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
/* /*
* Created by Christian Schabesberger on 23.08.15. * Created by Christian Schabesberger on 23.08.15.
* *
@ -22,54 +20,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
/** import org.schabi.newpipe.extractor.exceptions.ExtractionException;
* Provides access to the video streaming services supported by NewPipe.
* Currently only Youtube until the API becomes more stable.
*/
@SuppressWarnings("ALL") /**
* Provides access to streaming services supported by NewPipe.
*/
public class NewPipe { public class NewPipe {
private static final String TAG = NewPipe.class.toString(); private static final String TAG = NewPipe.class.toString();
private NewPipe() {
}
private static Downloader downloader = null; private static Downloader downloader = null;
public static StreamingService[] getServices() { private NewPipe() {
return ServiceList.serviceList;
}
public static StreamingService getService(int serviceId) throws ExtractionException {
for (StreamingService s : ServiceList.serviceList) {
if (s.getServiceId() == serviceId) {
return s;
}
}
return null;
}
public static StreamingService getService(String serviceName) throws ExtractionException {
return ServiceList.serviceList[getIdOfService(serviceName)];
}
public static String getNameOfService(int id) {
try {
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "";
}
}
public static int getIdOfService(String serviceName) {
for (int i = 0; i < ServiceList.serviceList.length; i++) {
if (ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
return i;
}
}
return -1;
} }
public static void init(Downloader d) { public static void init(Downloader d) {
@ -80,12 +40,63 @@ public class NewPipe {
return downloader; return downloader;
} }
public static StreamingService getServiceByUrl(String url) { /*//////////////////////////////////////////////////////////////////////////
for (StreamingService s : ServiceList.serviceList) { // Utils
if (s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) { //////////////////////////////////////////////////////////////////////////*/
return s;
public static StreamingService[] getServices() {
final ServiceList[] values = ServiceList.values();
final StreamingService[] streamingServices = new StreamingService[values.length];
for (int i = 0; i < values.length; i++) streamingServices[i] = values[i].getService();
return streamingServices;
}
public static StreamingService getService(int serviceId) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getServiceId() == serviceId) {
return item.getService();
} }
} }
return null; throw new ExtractionException("There's no service with the id = \"" + serviceId + "\"");
}
public static StreamingService getService(String serviceName) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getServiceInfo().name.equals(serviceName)) {
return item.getService();
}
}
throw new ExtractionException("There's no service with the name = \"" + serviceName + "\"");
}
public static StreamingService getServiceByUrl(String url) throws ExtractionException {
for (ServiceList item : ServiceList.values()) {
if (item.getService().getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
return item.getService();
}
}
throw new ExtractionException("No service can handle the url = \"" + url + "\"");
}
public static int getIdOfService(String serviceName) {
try {
//noinspection ConstantConditions
return getService(serviceName).getServiceId();
} catch (ExtractionException ignored) {
return -1;
}
}
public static String getNameOfService(int id) {
try {
//noinspection ConstantConditions
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "<unknown>";
}
} }
} }

View File

@ -1,15 +1,36 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/* /**
* Created by the-scrabi on 18.02.17. * A list of supported services.
*/ */
public enum ServiceList {
Youtube(new YoutubeService(0, "Youtube")),
SoundCloud(new SoundcloudService(1, "SoundCloud"));
// DailyMotion(new DailyMotionService(2, "DailyMotion"));
class ServiceList { private final StreamingService service;
public static final StreamingService[] serviceList = {
new YoutubeService(0), ServiceList(StreamingService service) {
new SoundcloudService(1) this.service = service;
}; }
public StreamingService getService() {
return service;
}
public StreamingService.ServiceInfo getServiceInfo() {
return service.getServiceInfo();
}
public int getId() {
return service.getServiceId();
}
@Override
public String toString() {
return service.getServiceInfo().name;
}
} }

View File

@ -10,7 +10,11 @@ import java.io.IOException;
public abstract class StreamingService { public abstract class StreamingService {
public class ServiceInfo { public class ServiceInfo {
public String name = ""; public final String name;
public ServiceInfo(String name) {
this.name = name;
}
} }
public enum LinkType { public enum LinkType {
@ -20,35 +24,46 @@ public abstract class StreamingService {
PLAYLIST PLAYLIST
} }
private int serviceId; private final int serviceId;
private final ServiceInfo serviceInfo;
public StreamingService(int id) { public StreamingService(int id, String name) {
serviceId = id; this.serviceId = id;
this.serviceInfo = new ServiceInfo(name);
} }
public abstract ServiceInfo getServiceInfo();
public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
public abstract UrlIdHandler getPlaylistUrlIdHandlerInstance();
public abstract SearchEngine getSearchEngineInstance();
public abstract SuggestionExtractor getSuggestionExtractorInstance();
public abstract StreamExtractor getStreamExtractorInstance(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException;
public abstract PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException;
public final int getServiceId() { public final int getServiceId() {
return serviceId; return serviceId;
} }
public ServiceInfo getServiceInfo() {
return serviceInfo;
}
public abstract UrlIdHandler getStreamUrlIdHandler();
public abstract UrlIdHandler getChannelUrlIdHandler();
public abstract UrlIdHandler getPlaylistUrlIdHandler();
public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null);
}
public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
return getPlaylistExtractor(url, null);
}
/** /**
* figure out where the link is pointing to (a channel, video, playlist, etc.) * figure out where the link is pointing to (a channel, video, playlist, etc.)
*/ */
public final LinkType getLinkTypeByUrl(String url) { public final LinkType getLinkTypeByUrl(String url) {
UrlIdHandler sH = getStreamUrlIdHandlerInstance(); UrlIdHandler sH = getStreamUrlIdHandler();
UrlIdHandler cH = getChannelUrlIdHandlerInstance(); UrlIdHandler cH = getChannelUrlIdHandler();
UrlIdHandler pH = getPlaylistUrlIdHandlerInstance(); UrlIdHandler pH = getPlaylistUrlIdHandler();
if (sH.acceptUrl(url)) { if (sH.acceptUrl(url)) {
return LinkType.STREAM; return LinkType.STREAM;

View File

@ -33,8 +33,7 @@ public abstract class SuggestionExtractor {
this.serviceId = serviceId; this.serviceId = serviceId;
} }
public abstract List<String> suggestionList(String query, String contentCountry) public abstract List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException;
throws ExtractionException, IOException;
public int getServiceId() { public int getServiceId() {
return serviceId; return serviceId;

View File

@ -1,9 +1,6 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import java.io.IOException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
/* /*
* Created by Christian Schabesberger on 26.07.16. * Created by Christian Schabesberger on 26.07.16.

View File

@ -1,11 +1,10 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException; import java.io.IOException;
@ -31,8 +30,13 @@ import java.io.IOException;
public abstract class ChannelExtractor extends ListExtractor { public abstract class ChannelExtractor extends ListExtractor {
public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(urlIdHandler, serviceId, url); super(service, url, nextStreamsUrl);
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getChannelUrlIdHandler();
} }
public abstract String getChannelId() throws ParsingException; public abstract String getChannelId() throws ParsingException;
@ -40,7 +44,6 @@ public abstract class ChannelExtractor extends ListExtractor {
public abstract String getAvatarUrl() throws ParsingException; public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract long getSubscriberCount() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException;
} }

View File

@ -1,11 +1,16 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List; import java.io.IOException;
import java.util.ArrayList;
/* /*
* Created by Christian Schabesberger on 31.07.16. * Created by Christian Schabesberger on 31.07.16.
@ -27,17 +32,36 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class ChannelInfo extends Info { public class ChannelInfo extends ListInfo {
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
}
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
return service.getChannelExtractor(null, nextStreamsUrl).getNextStreams();
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static ChannelInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getChannelExtractor(url));
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException { public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
ChannelInfo info = new ChannelInfo(); ChannelInfo info = new ChannelInfo();
// important data // important data
info.service_id = extractor.getServiceId(); info.service_id = extractor.getServiceId();
info.url = extractor.getUrl(); info.url = extractor.getCleanUrl();
info.id = extractor.getChannelId(); info.id = extractor.getChannelId();
info.name = extractor.getChannelName(); info.name = extractor.getChannelName();
info.has_more_streams = extractor.hasMoreStreams();
try { try {
info.avatar_url = extractor.getAvatarUrl(); info.avatar_url = extractor.getAvatarUrl();
@ -67,13 +91,16 @@ public class ChannelInfo extends Info {
info.errors.add(e); info.errors.add(e);
} }
// Lists can be null if a exception was thrown during extraction
if (info.related_streams == null) info.related_streams = new ArrayList<>();
info.has_more_streams = extractor.hasMoreStreams();
info.next_streams_url = extractor.getNextStreamsUrl();
return info; return info;
} }
public String avatar_url; public String avatar_url;
public String banner_url; public String banner_url;
public String feed_url; public String feed_url;
public List<InfoItem> related_streams;
public long subscriber_count = -1; public long subscriber_count = -1;
public boolean has_more_streams = false;
} }

View File

@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem {
public String thumbnail_url; public String thumbnail_url;
public String description; public String description;
public long subscriber_count = -1; public long subscriber_count = -1;
public long view_count = -1; public long stream_count = -1;
public ChannelInfoItem() { public ChannelInfoItem() {
super(InfoType.CHANNEL); super(InfoType.CHANNEL);

View File

@ -43,7 +43,7 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
addError(e); addError(e);
} }
try { try {
resultItem.view_count = extractor.getViewCount(); resultItem.stream_count = extractor.getStreamCount();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

View File

@ -28,5 +28,5 @@ public interface ChannelInfoItemExtractor {
String getWebPageUrl() throws ParsingException; String getWebPageUrl() throws ParsingException;
String getDescription() throws ParsingException; String getDescription() throws ParsingException;
long getSubscriberCount() throws ParsingException; long getSubscriberCount() throws ParsingException;
long getViewCount() throws ParsingException; long getStreamCount() throws ParsingException;
} }

View File

@ -1,18 +1,22 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException; import java.io.IOException;
public abstract class PlaylistExtractor extends ListExtractor { public abstract class PlaylistExtractor extends ListExtractor {
public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(urlIdHandler, serviceId, url); super(service, url, nextStreamsUrl);
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getPlaylistUrlIdHandler();
} }
public abstract String getPlaylistId() throws ParsingException; public abstract String getPlaylistId() throws ParsingException;
@ -22,6 +26,5 @@ public abstract class PlaylistExtractor extends ListExtractor {
public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderUrl() throws ParsingException;
public abstract String getUploaderName() throws ParsingException; public abstract String getUploaderName() throws ParsingException;
public abstract String getUploaderAvatarUrl() throws ParsingException; public abstract String getUploaderAvatarUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException; public abstract long getStreamCount() throws ParsingException;
public abstract long getStreamsCount() throws ParsingException;
} }

View File

@ -1,25 +1,49 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List; import java.io.IOException;
import java.util.ArrayList;
public class PlaylistInfo extends Info { public class PlaylistInfo extends ListInfo {
public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), nextStreamsUrl);
}
public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(null, nextStreamsUrl).getNextStreams();
}
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static PlaylistInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getPlaylistExtractor(url));
}
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException { public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
PlaylistInfo info = new PlaylistInfo(); PlaylistInfo info = new PlaylistInfo();
info.service_id = extractor.getServiceId(); info.service_id = extractor.getServiceId();
info.url = extractor.getUrl(); info.url = extractor.getCleanUrl();
info.id = extractor.getPlaylistId(); info.id = extractor.getPlaylistId();
info.name = extractor.getPlaylistName(); info.name = extractor.getPlaylistName();
info.has_more_streams = extractor.hasMoreStreams();
try { try {
info.streams_count = extractor.getStreamsCount(); info.stream_count = extractor.getStreamCount();
} catch (Exception e) { } catch (Exception e) {
info.errors.add(e); info.errors.add(e);
} }
@ -56,6 +80,11 @@ public class PlaylistInfo extends Info {
info.errors.add(e); info.errors.add(e);
} }
// Lists can be null if a exception was thrown during extraction
if (info.related_streams == null) info.related_streams = new ArrayList<>();
info.has_more_streams = extractor.hasMoreStreams();
info.next_streams_url = extractor.getNextStreamsUrl();
return info; return info;
} }
@ -64,7 +93,5 @@ public class PlaylistInfo extends Info {
public String uploader_url; public String uploader_url;
public String uploader_name; public String uploader_name;
public String uploader_avatar_url; public String uploader_avatar_url;
public long streams_count = 0; public long stream_count = 0;
public List<InfoItem> related_streams;
public boolean has_more_streams;
} }

View File

@ -8,7 +8,7 @@ public class PlaylistInfoItem extends InfoItem {
/** /**
* How many streams this playlist have * How many streams this playlist have
*/ */
public long streams_count = 0; public long stream_count = 0;
public PlaylistInfoItem() { public PlaylistInfoItem() {
super(InfoType.PLAYLIST); super(InfoType.PLAYLIST);

View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector { public class PlaylistInfoItemCollector extends InfoItemCollector {
@ -22,7 +21,7 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
addError(e); addError(e);
} }
try { try {
resultItem.streams_count = extractor.getStreamsCount(); resultItem.stream_count = extractor.getStreamCount();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

View File

@ -6,5 +6,5 @@ public interface PlaylistInfoItemExtractor {
String getThumbnailUrl() throws ParsingException; String getThumbnailUrl() throws ParsingException;
String getPlaylistName() throws ParsingException; String getPlaylistName() throws ParsingException;
String getWebPageUrl() throws ParsingException; String getWebPageUrl() throws ParsingException;
long getStreamsCount() throws ParsingException; long getStreamCount() throws ParsingException;
} }

View File

@ -59,7 +59,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
try { try {
result.resultList.add(streamCollector.extract(extractor)); result.resultList.add(streamCollector.extract(extractor));
} catch (FoundAdException ae) { } catch (FoundAdException ae) {
System.err.println("Found add"); System.err.println("Found ad");
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
@ -69,7 +69,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
try { try {
result.resultList.add(channelCollector.extract(extractor)); result.resultList.add(channelCollector.extract(extractor));
} catch (FoundAdException ae) { } catch (FoundAdException ae) {
System.err.println("Found add"); System.err.println("Found ad");
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

View File

@ -49,5 +49,5 @@ public abstract class SearchEngine {
//Result search(String query, int page); //Result search(String query, int page);
public abstract InfoItemSearchCollector search( public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter) String query, int page, String contentCountry, EnumSet<Filter> filter)
throws ExtractionException, IOException; throws IOException, ExtractionException;
} }

View File

@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.Vector;
/* /*
* Created by Christian Schabesberger on 29.02.16. * Created by Christian Schabesberger on 29.02.16.
@ -31,7 +31,7 @@ import java.util.Vector;
public class SearchResult { public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query, public static SearchResult getSearchResult(SearchEngine engine, String query,
int page, String languageCode, EnumSet<SearchEngine.Filter> filter) int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
throws ExtractionException, IOException { throws IOException, ExtractionException {
SearchResult result = engine SearchResult result = engine
.search(query, page, languageCode, filter) .search(query, page, languageCode, filter)
@ -50,6 +50,6 @@ public class SearchResult {
} }
public String suggestion; public String suggestion;
public List<InfoItem> resultList = new Vector<>(); public List<InfoItem> resultList = new ArrayList<>();
public List<Throwable> errors = new Vector<>(); public List<Throwable> errors = new ArrayList<>();
} }

View File

@ -1,37 +1,46 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor { public class SoundcloudChannelExtractor extends ChannelExtractor {
private String channelId; private String channelId;
private JSONObject channel; private JSONObject channel;
private String nextUrl;
public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(urlIdHandler, url, serviceId); super(service, url, nextStreamsUrl);
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
channelId = urlIdHandler.getId(url); channelId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId String apiUrl = "https://api.soundcloud.com/users/" + channelId +
+ "?client_id=" + SoundcloudParsingHelper.clientId(); "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
channel = new JSONObject(response); channel = new JSONObject(response);
} }
@Override
public String getCleanUrl() {
try {
return channel.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override @Override
public String getChannelId() { public String getChannelId() {
return channelId; return channelId;
@ -56,32 +65,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
} }
} }
@Override
public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"
+ "&offset=0"
+ "&linked_partitioning=1";
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
}
@Override @Override
public long getSubscriberCount() { public long getSubscriberCount() {
return channel.getLong("followers_count"); return channel.getLong("followers_count");
@ -93,26 +76,27 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
} }
@Override @Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
if (nextUrl.equals("")) { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getChannelId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Channel doesn't have more streams"); throw new ExtractionException("Channel doesn't have more streams");
} }
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Downloader dl = NewPipe.getDownloader(); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
String response = dl.download(nextUrl); return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
} }
} }

View File

@ -31,7 +31,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
} }
@Override @Override
public long getViewCount() { public long getStreamCount() {
return searchResult.getLong("track_count"); return searchResult.getLong("track_count");
} }

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -21,47 +18,28 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
@Override @Override
public String getUrl(String channelId) throws ParsingException { public String getUrl(String channelId) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + channelId);
String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }
@Override @Override
public String getId(String siteUrl) throws ParsingException { public String getId(String url) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
String response = dl.download(siteUrl);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(19);
return id;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }
@Override @Override
public String cleanUrl(String siteUrl) throws ParsingException { public String cleanUrl(String complexUrl) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(siteUrl); return ogElement.attr("content");
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
@ -71,6 +49,5 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
public boolean acceptUrl(String channelUrl) { public boolean acceptUrl(String channelUrl) {
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
return Parser.isMatch(regex, channelUrl.toLowerCase()); return Parser.isMatch(regex, channelUrl.toLowerCase());
} }
} }

View File

@ -1,13 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException; import org.json.JSONArray;
import java.text.ParseException; import org.json.JSONObject;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -15,61 +9,51 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class SoundcloudParsingHelper { public class SoundcloudParsingHelper {
private static String clientId;
private SoundcloudParsingHelper() { private SoundcloudParsingHelper() {
} }
public static final String clientId() throws ReCaptchaException, IOException, RegexException { public static String clientId() throws ReCaptchaException, IOException, RegexException {
if (clientId != null && !clientId.isEmpty()) return clientId;
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://soundcloud.com"); String response = dl.download("https://soundcloud.com");
Document doc = Jsoup.parse(response); Document doc = Jsoup.parse(response);
// TODO: Find a less heavy way to get the client_id
// Currently we are downloading a 1MB file (!) just to get the client_id,
// youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid.
// The embed mode has a way to get it, but we still have to download a heavy file (~800KB).
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first(); Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
String js = dl.download(jsElement.attr("src")); String js = dl.download(jsElement.attr("src"));
String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js); clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
return clientId; return clientId;
} }
public static String toTimeAgoString(String time) throws ParsingException {
try {
List<Long> times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30),
TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1),
TimeUnit.SECONDS.toMillis(1));
List<String> timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second");
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime();
StringBuilder timeAgoString = new StringBuilder();
for (int i = 0; i < times.size(); i++) {
Long current = times.get(i);
long currentAmount = timeAgo / current;
if (currentAmount > 0) {
timeAgoString.append(currentAmount).append(" ").append(timesString.get(i))
.append(currentAmount != 1 ? "s ago" : " ago");
break;
}
}
if (timeAgoString.toString().equals("")) {
timeAgoString.append("Just now");
}
return timeAgoString.toString();
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
}
public static String toDateString(String time) throws ParsingException { public static String toDateString(String time) throws ParsingException {
try { try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); Date date;
Date date = dateFormat.parse(time); // Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
} catch (Exception e) {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
}
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return newDateFormat.format(date); return newDateFormat.format(date);
} catch (ParseException e) { } catch (ParseException e) {
@ -77,4 +61,83 @@ public class SoundcloudParsingHelper {
} }
} }
/**
* Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
return new JSONObject(NewPipe.getDownloader().download(apiUrl));
}
/**
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).<br/>
*
* @return the url resolved
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"));
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
/**
* Fetch the embed player with the url and return the id (like the id from the json api).<br/>
*
* @return the id resolved
*/
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"));
return Parser.matchGroup1(",\"id\":(.*?),", response);
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always
* get MIN_ITEMS or more items.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
*/
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl);
}
return nextStreamsUrl;
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
}
String nextStreamsUrl;
try {
nextStreamsUrl = responseObject.getString("next_href");
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
nextStreamsUrl = "";
}
return nextStreamsUrl;
}
} }

View File

@ -1,38 +1,46 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId; private String playlistId;
private JSONObject playlist; private JSONObject playlist;
private List<String> nextTracks;
public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException { public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(urlIdHandler, url, serviceId); super(service, url, nextStreamsUrl);
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
playlistId = urlIdHandler.getId(url);
String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId playlistId = getUrlIdHandler().getId(getOriginalUrl());
+ "?client_id=" + SoundcloudParsingHelper.clientId(); String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
playlist = new JSONObject(response); playlist = new JSONObject(response);
} }
@Override
public String getCleanUrl() {
try {
return playlist.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override @Override
public String getPlaylistId() { public String getPlaylistId() {
return playlistId; return playlistId;
@ -69,61 +77,33 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
} }
@Override @Override
public long getStreamsCount() { public long getStreamCount() {
return playlist.getLong("track_count"); return playlist.getLong("track_count");
} }
@Override @Override
public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException { public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId // Note the "api", NOT "api-v2"
+ "?client_id=" + SoundcloudParsingHelper.clientId(); String apiUrl = "https://api.soundcloud.com/playlists/" + getPlaylistId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
String response = dl.download(apiUrl); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
try {
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
} catch (Exception e) {
nextTracks.add(track.getString("id"));
}
}
return collector; return collector;
} }
@Override @Override
public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException { public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (nextTracks.equals(null)) { if (!hasMoreStreams()) {
return null; throw new ExtractionException("Playlist doesn't have more streams");
} }
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Downloader dl = NewPipe.getDownloader(); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
// TODO: Do this per 10 tracks, instead of all tracks at once return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
String apiUrl = "https://api-v2.soundcloud.com/tracks?ids=";
for (String id : nextTracks) {
apiUrl += id;
if (!id.equals(nextTracks.get(nextTracks.size() - 1))) {
apiUrl += ",";
}
}
apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
nextTracks = null;
return collector;
} }
} }

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -21,13 +18,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override @Override
public String getUrl(String listId) throws ParsingException { public String getUrl(String listId) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + listId);
String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
@ -36,15 +27,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override @Override
public String getId(String url) throws ParsingException { public String getId(String url) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(23);
return id;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
@ -53,15 +36,10 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
@Override @Override
public String cleanUrl(String complexUrl) throws ParsingException { public String cleanUrl(String complexUrl) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(complexUrl); return ogElement.attr("content");
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }

View File

@ -1,9 +1,5 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
public class SoundcloudSearchEngine extends SearchEngine { public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8"; public static final String CHARSET_UTF_8 = "UTF-8";

View File

@ -13,61 +13,48 @@ import java.io.IOException;
public class SoundcloudService extends StreamingService { public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) { public SoundcloudService(int id, String name) {
super(id); super(id, name);
} }
@Override @Override
public ServiceInfo getServiceInfo() { public SearchEngine getSearchEngine() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Soundcloud";
return serviceInfo;
}
@Override
public StreamExtractor getStreamExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance();
if (urlIdHandler.acceptUrl(url)) {
return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId());
} else {
throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
return new SoundcloudSearchEngine(getServiceId()); return new SoundcloudSearchEngine(getServiceId());
} }
@Override @Override
public UrlIdHandler getStreamUrlIdHandlerInstance() { public UrlIdHandler getStreamUrlIdHandler() {
return SoundcloudStreamUrlIdHandler.getInstance(); return SoundcloudStreamUrlIdHandler.getInstance();
} }
@Override @Override
public UrlIdHandler getChannelUrlIdHandlerInstance() { public UrlIdHandler getChannelUrlIdHandler() {
return SoundcloudChannelUrlIdHandler.getInstance(); return SoundcloudChannelUrlIdHandler.getInstance();
} }
@Override @Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() { public UrlIdHandler getPlaylistUrlIdHandler() {
return SoundcloudPlaylistUrlIdHandler.getInstance(); return SoundcloudPlaylistUrlIdHandler.getInstance();
} }
@Override @Override
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException { public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId()); return new SoundcloudStreamExtractor(this, url);
} }
@Override @Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException { public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId()); return new SoundcloudChannelExtractor(this, url, nextStreamsUrl);
} }
@Override @Override
public SuggestionExtractor getSuggestionExtractorInstance() { public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new SoundcloudSuggestionExtractor(getServiceId()); return new SoundcloudSuggestionExtractor(getServiceId());
} }
} }

View File

@ -1,19 +1,14 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@ -21,33 +16,39 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor { public class SoundcloudStreamExtractor extends StreamExtractor {
private String pageUrl;
private String trackId;
private JSONObject track; private JSONObject track;
public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException { public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(urlIdHandler, pageUrl, serviceId); super(service, url);
}
Downloader dl = NewPipe.getDownloader(); @Override
public void fetchPage() throws IOException, ExtractionException {
trackId = urlIdHandler.getId(pageUrl); track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
track = new JSONObject(response);
if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) { if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy")); throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
} }
} }
@Override
public String getCleanUrl() {
try {
return track.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
}
@Override @Override
public String getId() { public String getId() {
return trackId; return track.getInt("id") + "";
} }
@Override @Override
@ -96,11 +97,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<AudioStream> getAudioStreams() throws ReCaptchaException, IOException, RegexException { public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
Vector<AudioStream> audioStreams = new Vector<>(); List<AudioStream> audioStreams = new ArrayList<>();
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams" String apiUrl = "https://api.soundcloud.com/i1/tracks/" + getId() + "/streams"
+ "?client_id=" + SoundcloudParsingHelper.clientId(); + "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
@ -113,20 +114,20 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<VideoStream> getVideoStreams() { public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
return null; return null;
} }
@Override @Override
public List<VideoStream> getVideoOnlyStreams() { public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
return null; return null;
} }
@Override @Override
public int getTimeStamp() throws ParsingException { public int getTimeStamp() throws ParsingException {
String timeStamp; String timeStamp;
try { try {
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
} catch (Parser.RegexException e) { } catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp // catch this instantly since an url does not necessarily have to have a time stamp
@ -190,16 +191,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemExtractor getNextVideo() { public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
return null; return null;
} }
@Override @Override
public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException { public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related" String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related"
+ "?client_id=" + SoundcloudParsingHelper.clientId(); + "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl); String response = dl.download(apiUrl);

View File

@ -35,7 +35,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override @Override
public String getUploadDate() throws ParsingException { public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at")); return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
} }
@Override @Override

View File

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -13,6 +10,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private SoundcloudStreamUrlIdHandler() { private SoundcloudStreamUrlIdHandler() {
} }
@ -23,13 +21,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override @Override
public String getUrl(String videoId) throws ParsingException { public String getUrl(String videoId) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + videoId);
String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
@ -38,15 +30,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override @Override
public String getId(String url) throws ParsingException { public String getId(String url) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(20);
return id;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
@ -55,15 +39,10 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
@Override @Override
public String cleanUrl(String complexUrl) throws ParsingException { public String cleanUrl(String complexUrl) throws ParsingException {
try { try {
Downloader dl = NewPipe.getDownloader(); Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
String response = dl.download(complexUrl); return ogElement.attr("content");
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }

View File

@ -1,10 +1,5 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
@ -13,6 +8,11 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor { public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8"; public static final String CHARSET_UTF_8 = "UTF-8";

View File

@ -43,7 +43,7 @@ public class ItagItem {
// Disable Opus codec as it's not well supported in older devices // Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, AUDIO, WEBMA, 50), // new ItagItem(249, AUDIO, WEBMA, 50),
// new ItagItem(250, AUDIO, WEBMA, 70), // new ItagItem(250, AUDIO, WEBMA, 70),
// new ItagItem(251, AUDIO, WEBMA, 16), // new ItagItem(251, AUDIO, WEBMA, 160),
new ItagItem(171, AUDIO, WEBMA, 128), new ItagItem(171, AUDIO, WEBMA, 128),
new ItagItem(172, AUDIO, WEBMA, 256), new ItagItem(172, AUDIO, WEBMA, 256),
new ItagItem(139, AUDIO, M4A, 48), new ItagItem(139, AUDIO, M4A, 48),

View File

@ -8,7 +8,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -44,6 +44,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor { public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc; private Document doc;
/** /**
@ -51,29 +52,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/ */
private Document nextStreamsAjax; private Document nextStreamsAjax;
/*////////////////////////////////////////////////////////////////////////// public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
// Variables for cache purposes (not "select" the current document all over again) super(service, url, nextStreamsUrl);
//////////////////////////////////////////////////////////////////////////*/ }
private String channelId;
private String channelName;
private String avatarUrl;
private String bannerUrl;
private String feedUrl;
private long subscriberCount = -1;
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { @Override
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId); public void fetchPage() throws IOException, ExtractionException {
fetchDocument(); Downloader downloader = NewPipe.getDownloader();
String userUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextStreamsUrl = getNextStreamsUrlFrom(doc);
nextStreamsAjax = null;
} }
@Override @Override
public String getChannelId() throws ParsingException { public String getChannelId() throws ParsingException {
try { try {
if (channelId == null) { return getUrlIdHandler().getId(getCleanUrl());
channelId = getUrlIdHandler().getId(getUrl());
}
return channelId;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get channel id"); throw new ParsingException("Could not get channel id");
} }
@ -82,11 +80,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getChannelName() throws ParsingException { public String getChannelName() throws ParsingException {
try { try {
if (channelName == null) { return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return channelName;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get channel name"); throw new ParsingException("Could not get channel name");
} }
@ -95,11 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
try { try {
if (avatarUrl == null) { return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
}
return avatarUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get avatar", e); throw new ParsingException("Could not get avatar", e);
} }
@ -108,59 +98,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
try { try {
if (bannerUrl == null) { Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); String cssContent = el.html();
String cssContent = el.html(); String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url; return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
}
return bannerUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get Banner", e); throw new ParsingException("Could not get Banner", e);
} }
} }
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul);
return collector;
}
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (subscriberCount == -1) { if (el != null) {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
if (el != null) { } else {
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text())); throw new ParsingException("Could not get subscriber count");
} else {
throw new ParsingException("Could not get subscriber count");
}
} }
return subscriberCount;
} }
@Override @Override
public String getFeedUrl() throws ParsingException { public String getFeedUrl() throws ParsingException {
try { try {
if (feedUrl == null) { String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id"); return channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
}
return feedUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get feed url", e); throw new ParsingException("Could not get feed url", e);
} }
} }
@Override @Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul);
return collector;
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasMoreStreams()) {
throw new ExtractionException("Channel doesn't have more streams"); throw new ExtractionException("Channel doesn't have more streams");
} }
@ -169,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return collector; return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@ -182,8 +160,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) { if (!nextStreamsHtmlDataRaw.isEmpty()) {
Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl); nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
} else { } else {
nextStreamsUrl = ""; nextStreamsUrl = "";
} }
@ -192,7 +169,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
} }
private String getNextStreamsUrl(Document d) throws ParsingException { private String getNextStreamsUrlFrom(Document d) throws ParsingException {
try { try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) { if (button != null) {
@ -206,17 +183,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
} }
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000";
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear(); collector.getItemList().clear();
@ -230,7 +196,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public boolean isAd() throws ParsingException { public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty(); return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
} }
@Override @Override
@ -259,7 +226,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public int getDuration() throws ParsingException { public int getDuration() throws ParsingException {
try { try {
return YoutubeParsingHelper.parseDurationString( return YoutubeParsingHelper.parseDurationString(
li.select("span[class=\"video-time\"]").first().text()); li.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) { } catch (Exception e) {
if (isLiveStream(li)) { if (isLiveStream(li)) {
// -1 for no duration // -1 for no duration

View File

@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
} }
@Override @Override
public long getViewCount() throws ParsingException { public long getStreamCount() throws ParsingException {
Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first(); Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
if (metaEl == null) { if (metaEl == null) {
return 0; return 0;

View File

@ -7,6 +7,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -23,39 +24,31 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc = null; private Document doc;
/** /**
* It's lazily initialized (when getNextStreams is called) * It's lazily initialized (when getNextStreams is called)
*/ */
private Document nextStreamsAjax = null; private Document nextStreamsAjax;
/*////////////////////////////////////////////////////////////////////////// public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
// Variables for cache purposes (not "select" the current document all over again) super(service, url, nextStreamsUrl);
//////////////////////////////////////////////////////////////////////////*/ }
private String playlistId;
private String playlistName;
private String avatarUrl;
private String bannerUrl;
private long streamsCount; @Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
private String uploaderUrl; String pageContent = downloader.download(getCleanUrl());
private String uploaderName; doc = Jsoup.parse(pageContent, getCleanUrl());
private String uploaderAvatarUrl;
public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException { nextStreamsUrl = getNextStreamsUrlFrom(doc);
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId); nextStreamsAjax = null;
fetchDocument();
} }
@Override @Override
public String getPlaylistId() throws ParsingException { public String getPlaylistId() throws ParsingException {
try { try {
if (playlistId == null) { return getUrlIdHandler().getId(getCleanUrl());
playlistId = getUrlIdHandler().getId(getUrl());
}
return playlistId;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist id"); throw new ParsingException("Could not get playlist id");
} }
@ -64,11 +57,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getPlaylistName() throws ParsingException { public String getPlaylistName() throws ParsingException {
try { try {
if (playlistName == null) { return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
}
return playlistName;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist name"); throw new ParsingException("Could not get playlist name");
} }
@ -77,11 +66,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
try { try {
if (avatarUrl == null) { return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
}
return avatarUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist avatar"); throw new ParsingException("Could not get playlist avatar");
} }
@ -90,18 +75,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
try { try {
if (bannerUrl == null) { Element el = doc.select("div[id=\"gh-banner\"] style").first();
Element el = doc.select("div[id=\"gh-banner\"] style").first(); String cssContent = el.html();
String cssContent = el.html(); String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent); if (url.contains("s.ytimg.com")) {
if (url.contains("s.ytimg.com")) { return null;
bannerUrl = null; } else {
} else { return url.substring(0, url.indexOf(");"));
bannerUrl = url.substring(0, url.indexOf(");"));
}
} }
return bannerUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist Banner"); throw new ParsingException("Could not get playlist Banner");
} }
@ -110,11 +93,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
if (uploaderUrl == null) { return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
}
return uploaderUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name"); throw new ParsingException("Could not get playlist uploader name");
} }
@ -123,11 +102,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
if (uploaderName == null) { return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return uploaderName;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name"); throw new ParsingException("Could not get playlist uploader name");
} }
@ -136,54 +111,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
if (uploaderAvatarUrl == null) { return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
}
return uploaderAvatarUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar"); throw new ParsingException("Could not get playlist uploader avatar");
} }
} }
@Override @Override
public long getStreamsCount() throws ParsingException { public long getStreamCount() throws ParsingException {
if (streamsCount <= 0) { String input;
String input;
try { try {
input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text(); input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
} catch (IndexOutOfBoundsException e) { } catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not get video count from playlist", e); throw new ParsingException("Could not get video count from playlist", e);
}
try {
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string
if (!input.isEmpty()) {
streamsCount = 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
} }
return streamsCount; try {
return Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string
if (!input.isEmpty()) {
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
} }
@Override @Override
public StreamInfoItemCollector getStreams() throws ParsingException { public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
collectStreamsFrom(collector, tbody); collectStreamsFrom(collector, tbody);
return collector; return collector;
} }
@Override @Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()){ if (!hasMoreStreams()) {
throw new ExtractionException("Playlist doesn't have more streams"); throw new ExtractionException("Playlist doesn't have more streams");
} }
@ -191,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return collector; return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@ -204,8 +171,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) { if (!nextStreamsHtmlDataRaw.isEmpty()) {
final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl); nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
} else { } else {
nextStreamsUrl = ""; nextStreamsUrl = "";
} }
@ -214,17 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
} }
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException { private String getNextStreamsUrlFrom(Document d) throws ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private String getNextStreamsUrl(Document d) throws ParsingException {
try { try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) { if (button != null) {
@ -241,7 +197,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear(); collector.getItemList().clear();
final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance(); final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
for (final Element li : element.children()) { for (final Element li : element.children()) {
collector.commit(new StreamInfoItemExtractor() { collector.commit(new StreamInfoItemExtractor() {
@Override @Override
@ -252,7 +208,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getWebPageUrl() throws ParsingException { public String getWebPageUrl() throws ParsingException {
try { try {
return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id")); return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e); throw new ParsingException("Could not get web page url for the video", e);
} }
@ -300,7 +256,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg"; return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }

View File

@ -34,61 +34,48 @@ import java.io.IOException;
public class YoutubeService extends StreamingService { public class YoutubeService extends StreamingService {
public YoutubeService(int id) { public YoutubeService(int id, String name) {
super(id); super(id, name);
} }
@Override @Override
public ServiceInfo getServiceInfo() { public SearchEngine getSearchEngine() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Youtube";
return serviceInfo;
}
@Override
public StreamExtractor getStreamExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
if (urlIdHandler.acceptUrl(url)) {
return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
} else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine(getServiceId()); return new YoutubeSearchEngine(getServiceId());
} }
@Override @Override
public UrlIdHandler getStreamUrlIdHandlerInstance() { public UrlIdHandler getStreamUrlIdHandler() {
return YoutubeStreamUrlIdHandler.getInstance(); return YoutubeStreamUrlIdHandler.getInstance();
} }
@Override @Override
public UrlIdHandler getChannelUrlIdHandlerInstance() { public UrlIdHandler getChannelUrlIdHandler() {
return YoutubeChannelUrlIdHandler.getInstance(); return YoutubeChannelUrlIdHandler.getInstance();
} }
@Override @Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() { public UrlIdHandler getPlaylistUrlIdHandler() {
return YoutubePlaylistUrlIdHandler.getInstance(); return YoutubePlaylistUrlIdHandler.getInstance();
} }
@Override @Override
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException { public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId()); return new YoutubeStreamExtractor(this, url);
} }
@Override @Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException { public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId()); return new YoutubeChannelExtractor(this, url, nextStreamsUrl);
} }
@Override @Override
public SuggestionExtractor getSuggestionExtractorInstance() { public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(this, url, nextStreamsUrl);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new YoutubeSuggestionExtractor(getServiceId()); return new YoutubeSuggestionExtractor(getServiceId());
} }
} }

View File

@ -10,7 +10,7 @@ import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -26,9 +26,9 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -80,12 +80,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/ /*//////////////////////////////////////////////////////////////////////////*/
private Document doc; private Document doc;
private final String dirtyUrl;
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException { public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId); super(service, url);
dirtyUrl = pageUrl;
fetchDocument();
} }
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
@ -95,7 +92,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getId() throws ParsingException { public String getId() throws ParsingException {
try { try {
return getUrlIdHandler().getId(getUrl()); return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get stream id"); throw new ParsingException("Could not get stream id");
} }
@ -238,8 +235,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<AudioStream> getAudioStreams() throws ParsingException { public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
Vector<AudioStream> audioStreams = new Vector<>(); List<AudioStream> audioStreams = new ArrayList<>();
try { try {
String encodedUrlMap; String encodedUrlMap;
// playerArgs could be null if the video is age restricted // playerArgs could be null if the video is age restricted
@ -288,8 +285,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<VideoStream> getVideoStreams() throws ParsingException { public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
Vector<VideoStream> videoStreams = new Vector<>(); List<VideoStream> videoStreams = new ArrayList<>();
try { try {
String encodedUrlMap; String encodedUrlMap;
@ -342,8 +339,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<VideoStream> getVideoOnlyStreams() throws ParsingException { public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
Vector<VideoStream> videoOnlyStreams = new Vector<>(); List<VideoStream> videoOnlyStreams = new ArrayList<>();
try { try {
String encodedUrlMap; String encodedUrlMap;
@ -405,7 +402,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException { public int getTimeStamp() throws ParsingException {
String timeStamp; String timeStamp;
try { try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl); timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
} catch (Parser.RegexException e) { } catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp // catch this instantly since an url does not necessarily have to have a time stamp
@ -516,7 +513,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemExtractor getNextVideo() throws ParsingException { public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
try { try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first()); .select("li").first());
@ -526,9 +523,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemCollector getRelatedVideos() throws ParsingException { public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
try { try {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[id=\"watch-related\"]").first(); Element ul = doc.select("ul[id=\"watch-related\"]").first();
if (ul != null) { if (ul != null) {
for (Element li : ul.children()) { for (Element li : ul.children()) {
@ -617,11 +614,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// cached values // cached values
private static volatile String decryptionCode = ""; private static volatile String decryptionCode = "";
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException { @Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl()); String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getUrl()); doc = Jsoup.parse(pageContent, getCleanUrl());
JSONObject ytPlayerConfig; JSONObject ytPlayerConfig;
String playerUrl; String playerUrl;
@ -632,7 +630,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Check if the video is age restricted // Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) { if (pageContent.contains("<meta property=\"og:restrictions:age")) {
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl()); playerUrl = getPlayerUrlFromRestrictedVideo();
isAgeRestricted = true; isAgeRestricted = true;
} else { } else {
ytPlayerConfig = getPlayerConfig(pageContent); ytPlayerConfig = getPlayerConfig(pageContent);
@ -683,7 +681,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not parse yt player config", e); throw new ParsingException("Could not parse yt player config", e);
} }
if (isLiveStream) { if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now."); throw new LiveStreamException("This is a Live stream. Can't use those right now.");
} }
return playerArgs; return playerArgs;
@ -709,12 +707,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException { private String getPlayerUrlFromRestrictedVideo() throws ParsingException, ReCaptchaException {
try { try {
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
String playerUrl = ""; String playerUrl = "";
String videoId = getUrlIdHandler().getId(pageUrl); String embedUrl = "https://www.youtube.com/embed/" + getId();
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl); String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1() //todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"); Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
@ -811,7 +808,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public boolean isAd() throws ParsingException { public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty(); return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
} }
@Override @Override
@ -829,8 +827,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public int getDuration() throws ParsingException { public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString( try {
li.select("span.video-time").first().text()); return YoutubeParsingHelper.parseDurationString(
li.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) {
if (isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
} }
@Override @Override
@ -845,12 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public long getViewCount() throws ParsingException { public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try { try {
return Long.parseLong(Utils.removeNonDigitCharacters( return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text())); li.select("span.view-count").first().text()));
@ -875,6 +876,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
return thumbnailUrl; return thumbnailUrl;
} }
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if (bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if (item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
}; };
} }
} }

View File

@ -59,7 +59,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public int getDuration() throws ParsingException { public int getDuration() throws ParsingException {
try { try {
return YoutubeParsingHelper.parseDurationString( return YoutubeParsingHelper.parseDurationString(
item.select("span[class=\"video-time\"]").first().text()); item.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) { } catch (Exception e) {
if (isLiveStream(item)) { if (isLiveStream(item)) {
// -1 for no duration // -1 for no duration
@ -104,16 +104,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (div == null) { if (div == null) {
return -1; return -1;
} else { } else {
input = div.select("li").get(1) input = div.select("li").get(1).text();
.text();
} }
} catch (IndexOutOfBoundsException e) { } catch (IndexOutOfBoundsException e) {
if (isLiveStream(item)) { if (isLiveStream(item)) {
// -1 for no view count // -1 for no view count
return -1; return -1;
} else { } else {
throw new ParsingException( throw new ParsingException("Could not parse yt-lockup-meta although available: " + getTitle(), e);
"Could not parse yt-lockup-meta although available: " + getTitle(), e);
} }
} }
@ -161,7 +159,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public boolean isAd() throws ParsingException { public boolean isAd() throws ParsingException {
return !item.select("span[class*=\"icon-not-available\"]").isEmpty(); return !item.select("span[class*=\"icon-not-available\"]").isEmpty() ||
!item.select("span[class*=\"yt-badge-ad\"]").isEmpty();
} }
private boolean isLiveStream(Element item) { private boolean isLiveStream(Element item) {

View File

@ -1,25 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONArray;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/* /*
* Created by Christian Schabesberger on 28.09.16. * Created by Christian Schabesberger on 28.09.16.
* *
@ -49,51 +41,24 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
} }
@Override @Override
public List<String> suggestionList( public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
String query, String contentCountry) Downloader dl = NewPipe.getDownloader();
throws ExtractionException, IOException {
List<String> suggestions = new ArrayList<>(); List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String url = "https://suggestqueries.google.com/complete/search" String url = "https://suggestqueries.google.com/complete/search"
+ "?client=" + "" + "?client=" + "firefox" // 'toolbar' for xml
+ "&output=" + "toolbar"
+ "&ds=" + "yt" + "&ds=" + "yt"
+ "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8) + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
String response = dl.download(url); String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try { try {
dBuilder = dbFactory.newDocumentBuilder(); JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1);
doc = dBuilder.parse(new InputSource( for (Object suggestion : suggestionsArray) suggestions.add(suggestion.toString());
new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
throw new ParsingException("Could not parse document.");
}
try {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
return suggestions;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get suggestions form document.", e); throw new ParsingException("Could not parse suggestions response.", e);
} }
return suggestions;
} }
} }

View File

@ -13,7 +13,7 @@ public abstract class Stream implements Serializable {
} }
/** /**
* Reveals whether two streams are the same, but have different urls * Reveals whether two streams have the same stats (format and bitrate, for example)
*/ */
public boolean equalStats(Stream cmp) { public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format; return cmp != null && format == cmp.format;

View File

@ -21,9 +21,10 @@ package org.schabi.newpipe.extractor.stream;
*/ */
import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -33,8 +34,14 @@ import java.util.List;
*/ */
public abstract class StreamExtractor extends Extractor { public abstract class StreamExtractor extends Extractor {
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) { public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(urlIdHandler, serviceId, url); super(service, url);
fetchPage();
}
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
return getService().getStreamUrlIdHandler();
} }
public abstract String getId() throws ParsingException; public abstract String getId() throws ParsingException;
@ -48,22 +55,22 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getUploadDate() throws ParsingException; public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException; public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws ParsingException, ReCaptchaException, IOException; public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoStreams() throws ParsingException; public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException; public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
public abstract String getDashMpdUrl() throws ParsingException; public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException; public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException; public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException; public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException; public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException; public abstract StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException; public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException;
public abstract StreamType getStreamType() throws ParsingException; public abstract StreamType getStreamType() throws ParsingException;
/** /**
* Analyses the webpage's document and extracts any error message there might be. * Analyses the webpage's document and extracts any error message there might be.
* *
* @return Error message; null if there is no error message. * @return Error message; null if there is no error message.
*/ */
public abstract String getErrorMessage(); public abstract String getErrorMessage();
} }

View File

@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Vector;
/* /*
* Created by Christian Schabesberger on 26.08.15. * Created by Christian Schabesberger on 26.08.15.
@ -46,11 +50,23 @@ public class StreamInfo extends Info {
public StreamInfo() { public StreamInfo() {
} }
public static StreamInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static StreamInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
return getInfo(serviceItem.getService(), url);
}
public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getStreamExtractor(url));
}
/** /**
* Fills out the video info fields which are common to all services. * Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses * Probably needs to be overridden by subclasses
*/ */
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException { public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo = new StreamInfo(); StreamInfo streamInfo = new StreamInfo();
try { try {
@ -80,7 +96,7 @@ public class StreamInfo extends Info {
// if one of these is not available an exception is meant to be thrown directly into the frontend. // if one of these is not available an exception is meant to be thrown directly into the frontend.
streamInfo.service_id = extractor.getServiceId(); streamInfo.service_id = extractor.getServiceId();
streamInfo.url = extractor.getUrl(); streamInfo.url = extractor.getCleanUrl();
streamInfo.stream_type = extractor.getStreamType(); streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = extractor.getId(); streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle(); streamInfo.name = extractor.getTitle();
@ -128,15 +144,12 @@ public class StreamInfo extends Info {
} }
// Lists can be null if a exception was thrown during extraction // Lists can be null if a exception was thrown during extraction
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>(); if (streamInfo.video_streams == null) streamInfo.video_streams = new ArrayList<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>(); if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>(); if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>();
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) { if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try { try {
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
// it has video, video only and audio streams and will only add to the list if it don't
// find a similar stream in the respective lists (calling Stream#equalStats).
DashMpdParser.getStreams(streamInfo); DashMpdParser.getStreams(streamInfo);
} catch (Exception e) { } catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the manifest, // Sometimes we receive 403 (forbidden) error when trying to download the manifest,
@ -246,6 +259,8 @@ public class StreamInfo extends Info {
streamInfo.addException(e); streamInfo.addException(e);
} }
if (streamInfo.related_streams == null) streamInfo.related_streams = new ArrayList<>();
return streamInfo; return streamInfo;
} }
@ -278,7 +293,7 @@ public class StreamInfo extends Info {
public int dislike_count = -1; public int dislike_count = -1;
public String average_rating; public String average_rating;
public StreamInfoItem next_video; public StreamInfoItem next_video;
public List<InfoItem> related_streams = new Vector<>(); public List<InfoItem> related_streams;
//in seconds. some metadata is not passed using a StreamInfo object! //in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0; public int start_position = 0;
} }

View File

@ -53,7 +53,13 @@ public class DashMpdParser {
} }
/** /**
* Download manifest and return nodelist with elements of tag "AdaptationSet" * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
* then it will search for any stream that the ItagItem has (by the id).
* <p>
* It has video, video only and audio streams and will only add to the list if it don't
* find a similar stream in the respective lists (calling {@link Stream#equalStats}).
*
* @param streamInfo where the parsed streams will be added
*/ */
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
String dashDoc; String dashDoc;

View File

@ -9,7 +9,7 @@ public class Utils {
* Remove all non-digit characters from a string.<p> * Remove all non-digit characters from a string.<p>
* Examples:<br/> * Examples:<br/>
* <ul><li>1 234 567 views -> 1234567</li> * <ul><li>1 234 567 views -> 1234567</li>
* <li>$ 31,133.124 -> 31133124</li></ul> * <li>$31,133.124 -> 31133124</li></ul>
* *
* @param toRemove string to remove non-digit chars * @param toRemove string to remove non-digit chars
* @return a string that contains only digits * @return a string that contains only digits
@ -21,15 +21,23 @@ public class Utils {
/** /**
* Check if throwable have the cause * Check if throwable have the cause
*/ */
public static boolean hasCauseThrowable(Throwable throwable, Class<?> causeToCheck) { public static boolean hasCauseThrowable(Throwable throwable, Class<?>... causesToCheck) {
// Check if getCause is not the same as cause (the getCause is already the root), // Check if getCause is not the same as cause (the getCause is already the root),
// as it will cause a infinite loop if it is // as it will cause a infinite loop if it is
Throwable cause, getCause = throwable; Throwable cause, getCause = throwable;
for (Class<?> causesEl : causesToCheck) {
if (throwable.getClass().isAssignableFrom(causesEl)) {
return true;
}
}
while ((cause = throwable.getCause()) != null && getCause != cause) { while ((cause = throwable.getCause()) != null && getCause != cause) {
getCause = cause; getCause = cause;
if (cause.getClass().isAssignableFrom(causeToCheck)) { for (Class<?> causesEl : causesToCheck) {
return true; if (cause.getClass().isAssignableFrom(causesEl)) {
return true;
}
} }
} }
return false; return false;

View File

@ -14,7 +14,7 @@ import java.util.Map;
import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.HttpsURLConnection;
/** /*
* Created by Christian Schabesberger on 28.01.16. * Created by Christian Schabesberger on 28.01.16.
* *
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -35,16 +35,17 @@ import javax.net.ssl.HttpsURLConnection;
*/ */
public class Downloader implements org.schabi.newpipe.extractor.Downloader { public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = ""; private static String mCookies = "";
private static Downloader instance = null; private static Downloader instance = null;
private Downloader() {} private Downloader() {
}
public static Downloader getInstance() { public static Downloader getInstance() {
if(instance == null) { if (instance == null) {
synchronized (Downloader.class) { synchronized (Downloader.class) {
if (instance == null) { if (instance == null) {
instance = new Downloader(); instance = new Downloader();
@ -62,11 +63,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return Downloader.mCookies; return Downloader.mCookies;
} }
/**Download the text file at the supplied URL as in download(String), /**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string. * but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of *
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language * @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/ * @return the contents of the specified text file
*/
public String download(String siteUrl, String language) throws IOException, ReCaptchaException { public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>(); Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language); requestProperties.put("Accept-Language", language);
@ -74,29 +78,35 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
} }
/**Download the text file at the supplied URL as in download(String), /**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string. * but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of *
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties * @param customProperties set request header properties
* @return the contents of the specified text file * @return the contents of the specified text file
* @throws IOException*/ * @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException { public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl); URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
Iterator it = customProperties.entrySet().iterator(); Iterator it = customProperties.entrySet().iterator();
while(it.hasNext()) { while (it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next(); Map.Entry pair = (Map.Entry) it.next();
con.setRequestProperty((String)pair.getKey(), (String)pair.getValue()); con.setRequestProperty((String) pair.getKey(), (String) pair.getValue());
} }
return dl(con); return dl(con);
} }
/**Common functionality between download(String url) and download(String url, String language)*/ /**
* Common functionality between download(String url) and download(String url, String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException { private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder(); StringBuilder response = new StringBuilder();
BufferedReader in = null; BufferedReader in = null;
try { try {
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET"); con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT); con.setRequestProperty("User-Agent", USER_AGENT);
@ -108,13 +118,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
new InputStreamReader(con.getInputStream())); new InputStreamReader(con.getInputStream()));
String inputLine; String inputLine;
while((inputLine = in.readLine()) != null) { while ((inputLine = in.readLine()) != null) {
response.append(inputLine); response.append(inputLine);
} }
} catch(UnknownHostException uhe) {//thrown when there's no internet connection } catch (UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe); throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) { } catch (Exception e) {
/* /*
* HTTP 429 == Too Many Request * HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request * Receive from Youtube.com = ReCaptcha challenge request
@ -123,9 +133,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
if (con.getResponseCode() == 429) { if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested"); throw new ReCaptchaException("reCaptcha Challenge requested");
} }
throw new IOException(e);
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally { } finally {
if(in != null) { if (in != null) {
in.close(); in.close();
} }
} }
@ -133,10 +144,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return response.toString(); return response.toString();
} }
/**Download (via HTTP) the text file located at the supplied URL, and return its contents. /**
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages. * Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download * @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/ * @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException { public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl); URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); HttpsURLConnection con = (HttpsURLConnection) url.openConnection();

View File

@ -0,0 +1,74 @@
package org.schabi.newpipe.extractor;
import org.junit.Test;
import java.util.HashSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
public class NewPipeTest {
@Test
public void getAllServicesTest() throws Exception {
assertEquals(NewPipe.getServices().length, ServiceList.values().length);
}
@Test
public void testAllServicesHaveDifferentId() throws Exception {
HashSet<Integer> servicesId = new HashSet<>();
for (StreamingService streamingService : NewPipe.getServices()) {
String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().name + ")";
assertTrue(errorMsg, servicesId.add(streamingService.getServiceId()));
}
}
@Test
public void getServiceWithId() throws Exception {
assertEquals(NewPipe.getService(Youtube.getId()), Youtube.getService());
assertEquals(NewPipe.getService(SoundCloud.getId()), SoundCloud.getService());
assertNotEquals(NewPipe.getService(SoundCloud.getId()), Youtube.getService());
}
@Test
public void getServiceWithName() throws Exception {
assertEquals(NewPipe.getService(Youtube.getServiceInfo().name), Youtube.getService());
assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().name), SoundCloud.getService());
assertNotEquals(NewPipe.getService(Youtube.getServiceInfo().name), SoundCloud.getService());
}
@Test
public void getServiceWithUrl() throws Exception {
assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), Youtube.getService());
assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), Youtube.getService());
assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), Youtube.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud.getService());
assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud.getService());
assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), Youtube.getService());
assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud.getService());
}
@Test
public void getIdWithServiceName() throws Exception {
assertEquals(NewPipe.getIdOfService(Youtube.getServiceInfo().name), Youtube.getId());
assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), SoundCloud.getId());
assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), Youtube.getId());
}
@Test
public void getServiceNameWithId() throws Exception {
assertEquals(NewPipe.getNameOfService(Youtube.getId()), Youtube.getServiceInfo().name);
assertEquals(NewPipe.getNameOfService(SoundCloud.getId()), SoundCloud.getServiceInfo().name);
assertNotEquals(NewPipe.getNameOfService(Youtube.getId()), SoundCloud.getServiceInfo().name);
}
}

View File

@ -1,16 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertTrue;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
/** import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 12.09.16. * Created by Christian Schabesberger on 12.09.16.
* *
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -41,8 +42,8 @@ public class YoutubeChannelExtractorTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube") extractor = Youtube.getService()
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); .getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
} }
@Test @Test
@ -61,7 +62,7 @@ public class YoutubeChannelExtractorTest {
} }
@Test @Test
public void testGetBannerurl() throws Exception { public void testGetBannerUrl() throws Exception {
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3")); assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3"));
} }
@ -81,9 +82,10 @@ public class YoutubeChannelExtractorTest {
} }
@Test @Test
public void testHasNextPage() throws Exception { public void testHasMoreStreams() throws Exception {
// this particular example (link) has a next page !!! // Setup the streams
assertTrue("no next page link found", extractor.hasMoreStreams()); extractor.getStreams();
assertTrue("don't have more streams", extractor.hasMoreStreams());
} }
@Test @Test
@ -92,16 +94,11 @@ public class YoutubeChannelExtractorTest {
} }
@Test @Test
public void testGetNextPage() throws Exception { public void testGetNextStreams() throws Exception {
extractor = NewPipe.getService("Youtube") // Setup the streams
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); extractor.getStreams();
assertTrue("next page didn't have content", !extractor.getStreams().getItemList().isEmpty()); assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
} }
@Test
public void testGetNextNextPageUrl() throws Exception {
extractor = NewPipe.getService("Youtube")
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
assertTrue("next page didn't have content", extractor.hasMoreStreams());
}
} }

View File

@ -0,0 +1,98 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
* Test for {@link PlaylistExtractor}
*/
public class YoutubePlaylistExtractorTest {
private PlaylistExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = Youtube.getService()
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
}
@Test
public void testGetDownloader() throws Exception {
assertNotNull(NewPipe.getDownloader());
}
@Test
public void testGetId() throws Exception {
assertEquals(extractor.getPlaylistId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
}
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getPlaylistName(), "important videos");
}
@Test
public void testGetAvatarUrl() throws Exception {
assertTrue(extractor.getAvatarUrl(), extractor.getAvatarUrl().contains("yt"));
}
@Test
public void testGetBannerUrl() throws Exception {
assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt"));
}
@Test
public void testGetUploaderUrl() throws Exception {
assertTrue(extractor.getUploaderUrl(), extractor.getUploaderUrl().contains("youtube.com"));
}
@Test
public void testGetUploaderName() throws Exception {
assertTrue(extractor.getUploaderName(), !extractor.getUploaderName().isEmpty());
}
@Test
public void testGetUploaderAvatarUrl() throws Exception {
assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("yt"));
}
@Test
public void testGetStreamsCount() throws Exception {
assertTrue("error in the streams count", extractor.getStreamCount() > 100);
}
@Test
public void testGetStreams() throws Exception {
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
}
@Test
public void testGetStreamsErrors() throws Exception {
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
}
@Test
public void testHasMoreStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have more streams", extractor.hasMoreStreams());
}
@Test
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
}
}

View File

@ -9,11 +9,12 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet; import java.util.EnumSet;
import static junit.framework.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static junit.framework.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/** /*
* Created by Christian Schabesberger on 29.12.15. * Created by Christian Schabesberger on 29.12.15.
* *
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -42,7 +43,7 @@ public class YoutubeSearchEngineAllTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "asdf" instead of "asdgff" // Youtube will suggest "asdf" instead of "asdgff"
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")

View File

@ -10,12 +10,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet; import java.util.EnumSet;
import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static junit.framework.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static junit.framework.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/** /*
* Created by Christian Schabesberger on 29.12.15. * Created by Christian Schabesberger on 29.12.15.
* *
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -44,7 +45,7 @@ public class YoutubeSearchEngineChannelTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "gronkh" instead of "grrunkh" // Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")
@ -59,7 +60,9 @@ public class YoutubeSearchEngineChannelTest {
@Test @Test
public void testChannelItemType() { public void testChannelItemType() {
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.CHANNEL); for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.CHANNEL, infoItem.info_type);
}
} }
@Test @Test

View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
@ -10,12 +11,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet; import java.util.EnumSet;
import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static junit.framework.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static junit.framework.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/** /*
* Created by Christian Schabesberger on 29.12.15. * Created by Christian Schabesberger on 29.12.15.
* *
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -44,7 +46,7 @@ public class YoutubeSearchEngineStreamTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); SearchEngine engine = Youtube.getService().getSearchEngine();
// Youtube will suggest "results" instead of "rsults", // Youtube will suggest "results" instead of "rsults",
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")
@ -58,8 +60,10 @@ public class YoutubeSearchEngineStreamTest {
} }
@Test @Test
public void testChannelItemType() { public void testStreamItemType() {
assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.STREAM); for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.STREAM, infoItem.info_type);
}
} }
@Test @Test

View File

@ -1,21 +1,23 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static junit.framework.Assert.assertTrue;
import java.io.IOException;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
/** import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 30.12.15. * Created by Christian Schabesberger on 30.12.15.
* *
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@ -45,8 +47,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube") extractor = Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
.getStreamExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A");
} }
@Test @Test
@ -56,10 +57,8 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetValidTimeStamp() throws ExtractionException, IOException { public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = StreamExtractor extractor = Youtube.getService().getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
NewPipe.getService("Youtube")
.getStreamExtractorInstance("https://youtu.be/FmG385_uUys?t=174");
assertTrue(Integer.toString(extractor.getTimeStamp()), assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174); extractor.getTimeStamp() == 174);
} }
@ -113,12 +112,12 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetAudioStreams() throws ParsingException, ReCaptchaException, IOException { public void testGetAudioStreams() throws IOException, ExtractionException {
assertTrue(!extractor.getAudioStreams().isEmpty()); assertTrue(!extractor.getAudioStreams().isEmpty());
} }
@Test @Test
public void testGetVideoStreams() throws ParsingException { public void testGetVideoStreams() throws IOException, ExtractionException {
for(VideoStream s : extractor.getVideoStreams()) { for(VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url, assertTrue(s.url,
s.url.contains(HTTPS)); s.url.contains(HTTPS));
@ -138,4 +137,11 @@ public class YoutubeStreamExtractorDefaultTest {
assertTrue(extractor.getDashMpdUrl(), assertTrue(extractor.getDashMpdUrl(),
extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty()); extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty());
} }
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos();
assertFalse(relatedVideos.getItemList().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
} }

View File

@ -0,0 +1,53 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 30.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeVideoExtractorGema.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* This exception is only thrown in Germany.
*
* WARNING: Deactivate this Test Case before uploading it to Github, otherwise CI will fail.
*/
@Ignore
public class YoutubeStreamExtractorGemaTest {
@Test
public void testGemaError() throws IOException, ExtractionException {
try {
NewPipe.init(Downloader.getInstance());
Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
fail("GemaException should be thrown");
} catch (YoutubeStreamExtractor.GemaException ignored) {
// Exception was thrown, Gema error detection is working.
}
}
}

View File

@ -0,0 +1,108 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/**
* Test for {@link YoutubeStreamUrlIdHandler}
*/
public class YoutubeStreamExtractorRestrictedTest {
public static final String HTTPS = "https://";
private StreamExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = Youtube.getService()
.getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor= Youtube.getService()
.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
@Test
public void testGetAgeLimit() throws ParsingException {
assertTrue(extractor.getAgeLimit() == 18);
}
@Test
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
@Test
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains(HTTPS));
}
@Test
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains(HTTPS));
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audiostream not always necessary
assertTrue(!extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
for(VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains(HTTPS));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.format),
0 <= s.format && s.format <= 4);
}
}
}

View File

@ -0,0 +1,117 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertTrue;
/**
* Test for {@link YoutubeStreamUrlIdHandler}
*/
public class YoutubeStreamUrlIdHandlerTest {
private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew";
private YoutubeStreamUrlIdHandler urlIdHandler;
@Before
public void setUp() throws Exception {
urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
NewPipe.init(Downloader.getInstance());
}
@Test(expected = NullPointerException.class)
public void getIdWithNullAsUrl() throws ParsingException {
urlIdHandler.getId(null);
}
@Test(expected = FoundAdException.class)
public void getIdForAd() throws ParsingException {
urlIdHandler.getId(AD_URL);
}
@Test
public void getIdForInvalidUrls() throws ParsingException {
List<String> invalidUrls = new ArrayList<>(50);
invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
invalidUrls.add("https://www.youtube.com/");
for(String invalidUrl: invalidUrls) {
Throwable exception = null;
try {
urlIdHandler.getId(invalidUrl);
} catch (ParsingException e) {
exception = e;
}
if(exception == null) {
fail("Expected ParsingException for url: " + invalidUrl);
}
}
}
@Test
public void getId() throws Exception {
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
// Shared links
String sharedId = "7JIArTByb3E";
String realId = "Q7JsK50NGaA";
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
}
@Test
public void testAcceptUrl() {
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s"));
//assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
String sharedId = "8A940MXKFmQ";
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId));
}
}

View File

@ -0,0 +1,51 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.schabi.newpipe.extractor.ServiceList.Youtube;
/*
* Created by Christian Schabesberger on 18.11.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeSuggestionExtractorTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SuggestionExtractor}
*/
public class YoutubeSuggestionExtractorTest {
private SuggestionExtractor suggestionExtractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
suggestionExtractor = Youtube.getService().getSuggestionExtractor();
}
@Test
public void testIfSuggestions() throws IOException, ExtractionException {
assertFalse(suggestionExtractor.suggestionList("hello", "de").isEmpty());
}
}