be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

made getPage() function be stand alone

This commit is contained in:
Christian Schabesberger 2018-02-26 15:55:27 +01:00 committed by Mauricio Colli
parent 4366b7321f
commit d812a114c5
26 changed files with 223 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -10,32 +10,26 @@ import java.util.List;
* Base class to extractors that have a list (e.g. playlists, users).
*/
public abstract class ListExtractor extends Extractor {
protected String nextPageUrl;
/**
* Get a new ListExtractor with the given nextPageUrl set.
*/
public ListExtractor(StreamingService service, String url, String nextPageUrl) throws ExtractionException {
public ListExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
setNextPageUrl(nextPageUrl);
}
@Nonnull
public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException;
public abstract InfoItemPage getInfoItemPage() throws IOException, ExtractionException;
public abstract String getNextPageUrl() throws IOException, ExtractionException;
public boolean hasNextPage() {
return nextPageUrl != null && !nextPageUrl.isEmpty();
public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException;
public boolean hasNextPage() throws IOException, ExtractionException {
return getNextPageUrl() != null && !getNextPageUrl().isEmpty();
}
public String getNextPageUrl() {
return nextPageUrl;
}
public void setNextPageUrl(String nextPageUrl) {
this.nextPageUrl = nextPageUrl;
}
/*//////////////////////////////////////////////////////////////////////////
// Inner

14
src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -68,20 +68,12 @@ public abstract class StreamingService {
public abstract UrlIdHandler getPlaylistUrlIdHandler();
public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract StreamExtractor getStreamExtractor(String url) throws ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url) throws ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException;
public abstract SubscriptionExtractor getSubscriptionExtractor();
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null);
}
public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
return getPlaylistExtractor(url, null);
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/

5
src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -31,9 +31,9 @@ import java.io.IOException;
public abstract class ChannelExtractor extends ListExtractor {
public ChannelExtractor(StreamingService service, String url, String nextPageUrl)
public ChannelExtractor(StreamingService service, String url)
throws ExtractionException {
super(service, url, nextPageUrl);
super(service, url);
}
@Nonnull
@ -55,5 +55,4 @@ public abstract class ChannelExtractor extends ListExtractor {
public abstract String getFeedUrl() throws ParsingException;
public abstract long getSubscriberCount() throws ParsingException;
public abstract String getDescription() throws ParsingException;
}

6
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -37,9 +37,9 @@ public class ChannelInfo extends ListInfo {
}
public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl)
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl)
throws IOException, ExtractionException {
return service.getChannelExtractor(url, nextPageUrl).getInfoItemPage();
return service.getChannelExtractor(url).getPage(pageUrl);
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@ -52,7 +52,7 @@ public class ChannelInfo extends ListInfo {
return getInfo(extractor);
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
// important data
int serviceId = extractor.getServiceId();

3
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -34,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor {
public KioskExtractor(StreamingService streamingService,
String url,
String nextPageUrl,
String kioskId)
throws ExtractionException {
super(streamingService, url, nextPageUrl);
super(streamingService, url);
this.id = kioskId;
}

6
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -37,12 +37,12 @@ public class KioskInfo extends ListInfo {
public static ListExtractor.InfoItemPage getMoreItems(StreamingService service,
String url,
String nextPageUrl,
String pageUrl,
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, nextPageUrl);
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
extractor.setContentCountry(contentCountry);
return extractor.getInfoItemPage();
return extractor.getPage(pageUrl);
}
public static KioskInfo getInfo(String url,

4
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -14,7 +14,6 @@ public class KioskList {
public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url,
final String nextPageUrl,
final String kioskId)
throws ExtractionException, IOException;
}
@ -74,8 +73,7 @@ public class KioskList {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handler.getUrl(kioskId),
nextPageUrl, kioskId);
ke.handler.getUrl(kioskId), kioskId);
}
}

4
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -14,8 +14,8 @@ import java.io.IOException;
public abstract class PlaylistExtractor extends ListExtractor {
public PlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public PlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Nonnull

7
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException;
@ -17,8 +16,8 @@ public class PlaylistInfo extends ListInfo {
super(serviceId, id, url, name);
}
public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url, nextPageUrl).getInfoItemPage();
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url).getPage(pageUrl);
}
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
@ -36,7 +35,7 @@ public class PlaylistInfo extends ListInfo {
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();

39
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -18,8 +18,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -80,32 +83,50 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return user.getString("description", "");
}
@Nonnull
@Override
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
public StreamInfoItemsCollector getStreams() throws ExtractionException {
if(streamInfoItemsCollector == null) {
computeNextPageAndGetStreams();
}
return streamInfoItemsCollector;
}
@Override
public String getNextPageUrl() throws ExtractionException {
if(nextPageUrl == null) {
computeNextPageAndGetStreams();
}
return nextPageUrl;
}
private void computeNextPageAndGetStreams() throws ExtractionException {
try {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
} catch (Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
return new InfoItemPage(collector, nextPageUrl);
}

36
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -4,10 +4,12 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.schabi.newpipe.extractor.Collector;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -16,9 +18,12 @@ import javax.annotation.Nonnull;
public class SoundcloudChartsExtractor extends KioskExtractor {
private String url;
public SoundcloudChartsExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
throws ExtractionException {
super(service, url, nextPageUrl, kioskId);
super(service, url, kioskId);
this.url = url;
}
@ -39,21 +44,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Chart doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl, true);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
return new InfoItemPage(collector, nextPageUrl);
}
@Nonnull
@Override
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
private void computNextPageAndStreams() throws IOException, ExtractionException {
collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
@ -72,6 +76,22 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if(nextPageUrl == null) {
computNextPageAndStreams();
}
return nextPageUrl;
}
@Nonnull
@Override
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
if(collector == null) {
computNextPageAndStreams();
}
return collector;
}
}

31
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -18,8 +18,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudPlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -89,7 +92,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(streamInfoItemsCollector == null) {
computeStreamsAndNextPageUrl();
}
return streamInfoItemsCollector;
}
private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
// Note the "api", NOT "api-v2"
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
@ -97,18 +107,25 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public String getNextPageUrl() throws IOException, ExtractionException {
if(nextPageUrl == null) {
computeStreamsAndNextPageUrl();
}
return nextPageUrl;
}
@Override
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
return new InfoItemPage(collector, nextPageUrl);
}

14
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -45,18 +45,18 @@ public class SoundcloudService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return new SoundcloudStreamExtractor(this, url);
}
@Override
public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudChannelExtractor(this, url, nextPageUrl);
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return new SoundcloudChannelExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(this, url, nextPageUrl);
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return new SoundcloudPlaylistExtractor(this, url);
}
@Override
@ -70,12 +70,10 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String nextPageUrl,
String id)
throws ExtractionException, IOException {
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
url,
nextPageUrl,
id);
}
};

2
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -18,7 +18,7 @@ import java.util.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public SoundcloudStreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}

57
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -50,7 +50,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
/**
* It's lazily initialized (when getInfoItemPage is called)
*/
private Document nextStreamsAjax;
// private Document nextStreamsAjax;
/**
* Unfortunately, we have to fetch the page even if we are only getting next streams,
@ -58,12 +58,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
* <br/>
* This help us to keep track on what are we fetching.
*/
private boolean fetchingNextStreams;
//private boolean fetchingNextStreams;
public YoutubeChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public YoutubeChannelExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
//fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
}
@Override
@ -72,10 +72,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);
if (!fetchingNextStreams) {
nextPageUrl = getNextPageUrlFrom(doc);
//nextStreamsAjax = null;
}
nextStreamsAjax = null;
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(doc);
}
@Nonnull
@ -171,7 +173,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
try {
if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams");
}
@ -180,28 +184,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextPageAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
final JsonObject ajaxJson = JsonParser.object().from(
NewPipe.getDownloader()
.download(pageUrl));
return new InfoItemPage(collector, nextPageUrl);
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
collectStreamsFrom(collector, ajaxHtml.select("body").first());
return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse json data for next streams", pe);
}
}
private void setupNextPageAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextPageUrl);
try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
throws ParsingException {
String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
if (!loadMoreHtmlDataRaw.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));
} else {
nextPageUrl = "";
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
return "";
}
}

55
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -29,10 +29,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
/**
* It's lazily initialized (when getInfoItemPage is called)
*/
private Document nextStreamsAjax;
private Document nextPageAjax;
public YoutubePlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public YoutubePlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -40,8 +40,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
nextPageUrl = getNextPageUrlFrom(doc);
nextStreamsAjax = null;
nextPageAjax = null;
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(doc);
}
@Nonnull
@ -139,34 +143,37 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
try {
if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
// setupNextStreamsAjax(NewPipe.getDownloader());
final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader()
.download(pageUrl));
final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
+ pageJson.getString("content_html")
+ "</tbody></table>", pageUrl);
return new InfoItemPage(collector, nextPageUrl);
collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse ajax json", pe);
}
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextPageUrl);
try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
throws ParsingException{
String nextPageHtml = pageJson.getString("load_more_widget_html");
if (!nextPageHtml.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
} else {
nextPageUrl = "";
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
return "";
}
}

16
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -65,18 +65,18 @@ public class YoutubeService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return new YoutubeStreamExtractor(this, url);
}
@Override
public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubeChannelExtractor(this, url, nextPageUrl);
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return new YoutubeChannelExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(this, url, nextPageUrl);
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return new YoutubePlaylistExtractor(this, url);
}
@Override
@ -92,9 +92,9 @@ public class YoutubeService extends StreamingService {
try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextPageUrl, String id)
throws ExtractionException, IOException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, nextPageUrl, id);
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
}
}, new YoutubeTrendingUrlIdHandler(), "Trending");
list.setDefaultKiosk("Trending");

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -85,7 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public YoutubeStreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}

11
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
View File

@ -37,9 +37,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
throws ExtractionException {
super(service, url, nextPageUrl, kioskId);
super(service, url, kioskId);
}
@Override
@ -61,7 +61,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
}
@Override
public ListExtractor.InfoItemPage getInfoItemPage() {
public String getNextPageUrl() {
return "";
}
@Override
public ListExtractor.InfoItemPage getPage(String pageUrl) {
return null;
}

4
src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -39,13 +39,13 @@ public abstract class StreamExtractor extends Extractor {
public static final int NO_AGE_LIMIT = 0;
public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public StreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Nonnull
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
protected UrlIdHandler getUrlIdHandler() {
return getService().getStreamUrlIdHandler();
}

2
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -73,7 +73,7 @@ public class SoundcloudChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

4
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -76,8 +76,8 @@ public class SoundcloudChartsExtractorTest {
@Test
public void testGetNextStreams() throws Exception {
extractor.getInfoItems();
assertFalse("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getInfoItemPage().infoItemList.isEmpty());
assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|| extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty());
}
@Test

2
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -90,7 +90,7 @@ public class SoundcloudPlaylistExtractorTest {
extractor.getStreams();
// This playlist don't have more streams, it should throw an error
extractor.getInfoItemPage();
extractor.getPage(extractor.getNextPageUrl());
fail("Expected exception wasn't thrown");
}

2
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -112,7 +112,7 @@ public class YoutubeChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors);
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

2
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -111,7 +111,7 @@ public class YoutubePlaylistExtractorTest {
public void testGetNextPage() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage infoItemPage = extractor.getInfoItemPage();
ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors);
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

4
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -85,8 +85,8 @@ public class YoutubeTrendingExtractorTest {
@Test
public void testGetNextStreams() throws Exception {
assertTrue("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getInfoItemPage().getNextItemsList().isEmpty());
assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|| extractor.getPage(extractor.getNextPageUrl()).getNextItemsList().isEmpty());
}
@Test