made getPage() function be stand alone

This commit is contained in:
Christian Schabesberger 2018-02-26 15:55:27 +01:00 committed by Mauricio Colli
parent 4366b7321f
commit d812a114c5
26 changed files with 223 additions and 171 deletions

View File

@ -10,32 +10,26 @@ import java.util.List;
* Base class to extractors that have a list (e.g. playlists, users).
*/
public abstract class ListExtractor extends Extractor {
protected String nextPageUrl;
/**
* Get a new ListExtractor with the given nextPageUrl set.
*/
public ListExtractor(StreamingService service, String url, String nextPageUrl) throws ExtractionException {
public ListExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
setNextPageUrl(nextPageUrl);
}
@Nonnull
public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException;
public abstract InfoItemPage getInfoItemPage() throws IOException, ExtractionException;
public abstract String getNextPageUrl() throws IOException, ExtractionException;
public boolean hasNextPage() {
return nextPageUrl != null && !nextPageUrl.isEmpty();
public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException;
public boolean hasNextPage() throws IOException, ExtractionException {
return getNextPageUrl() != null && !getNextPageUrl().isEmpty();
}
public String getNextPageUrl() {
return nextPageUrl;
}
public void setNextPageUrl(String nextPageUrl) {
this.nextPageUrl = nextPageUrl;
}
/*//////////////////////////////////////////////////////////////////////////
// Inner

View File

@ -68,20 +68,12 @@ public abstract class StreamingService {
public abstract UrlIdHandler getPlaylistUrlIdHandler();
public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract StreamExtractor getStreamExtractor(String url) throws ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url) throws ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException;
public abstract SubscriptionExtractor getSubscriptionExtractor();
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null);
}
public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
return getPlaylistExtractor(url, null);
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/

View File

@ -31,9 +31,9 @@ import java.io.IOException;
public abstract class ChannelExtractor extends ListExtractor {
public ChannelExtractor(StreamingService service, String url, String nextPageUrl)
public ChannelExtractor(StreamingService service, String url)
throws ExtractionException {
super(service, url, nextPageUrl);
super(service, url);
}
@Nonnull
@ -55,5 +55,4 @@ public abstract class ChannelExtractor extends ListExtractor {
public abstract String getFeedUrl() throws ParsingException;
public abstract long getSubscriberCount() throws ParsingException;
public abstract String getDescription() throws ParsingException;
}

View File

@ -37,9 +37,9 @@ public class ChannelInfo extends ListInfo {
}
public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl)
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl)
throws IOException, ExtractionException {
return service.getChannelExtractor(url, nextPageUrl).getInfoItemPage();
return service.getChannelExtractor(url).getPage(pageUrl);
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@ -52,7 +52,7 @@ public class ChannelInfo extends ListInfo {
return getInfo(extractor);
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
// important data
int serviceId = extractor.getServiceId();

View File

@ -34,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor {
public KioskExtractor(StreamingService streamingService,
String url,
String nextPageUrl,
String kioskId)
throws ExtractionException {
super(streamingService, url, nextPageUrl);
super(streamingService, url);
this.id = kioskId;
}

View File

@ -37,12 +37,12 @@ public class KioskInfo extends ListInfo {
public static ListExtractor.InfoItemPage getMoreItems(StreamingService service,
String url,
String nextPageUrl,
String pageUrl,
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, nextPageUrl);
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
extractor.setContentCountry(contentCountry);
return extractor.getInfoItemPage();
return extractor.getPage(pageUrl);
}
public static KioskInfo getInfo(String url,

View File

@ -14,7 +14,6 @@ public class KioskList {
public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url,
final String nextPageUrl,
final String kioskId)
throws ExtractionException, IOException;
}
@ -74,8 +73,7 @@ public class KioskList {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handler.getUrl(kioskId),
nextPageUrl, kioskId);
ke.handler.getUrl(kioskId), kioskId);
}
}

View File

@ -14,8 +14,8 @@ import java.io.IOException;
public abstract class PlaylistExtractor extends ListExtractor {
public PlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public PlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Nonnull

View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException;
@ -17,8 +16,8 @@ public class PlaylistInfo extends ListInfo {
super(serviceId, id, url, name);
}
public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url, nextPageUrl).getInfoItemPage();
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url).getPage(pageUrl);
}
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
@ -36,7 +35,7 @@ public class PlaylistInfo extends ListInfo {
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();

View File

@ -18,8 +18,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -80,32 +83,50 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return user.getString("description", "");
}
@Nonnull
@Override
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
public StreamInfoItemsCollector getStreams() throws ExtractionException {
if(streamInfoItemsCollector == null) {
computeNextPageAndGetStreams();
}
return streamInfoItemsCollector;
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public String getNextPageUrl() throws ExtractionException {
if(nextPageUrl == null) {
computeNextPageAndGetStreams();
}
return nextPageUrl;
}
private void computeNextPageAndGetStreams() throws ExtractionException {
try {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
} catch (Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
return new InfoItemPage(collector, nextPageUrl);
}

View File

@ -4,10 +4,12 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.schabi.newpipe.extractor.Collector;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -16,9 +18,12 @@ import javax.annotation.Nonnull;
public class SoundcloudChartsExtractor extends KioskExtractor {
private String url;
public SoundcloudChartsExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
throws ExtractionException {
super(service, url, nextPageUrl, kioskId);
super(service, url, kioskId);
this.url = url;
}
@ -39,21 +44,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Chart doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl, true);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
return new InfoItemPage(collector, nextPageUrl);
}
@Nonnull
@Override
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
private void computNextPageAndStreams() throws IOException, ExtractionException {
collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
@ -72,6 +76,22 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if(nextPageUrl == null) {
computNextPageAndStreams();
}
return nextPageUrl;
}
@Nonnull
@Override
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
if(collector == null) {
computNextPageAndStreams();
}
return collector;
}
}

View File

@ -18,8 +18,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudPlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -89,7 +92,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(streamInfoItemsCollector == null) {
computeStreamsAndNextPageUrl();
}
return streamInfoItemsCollector;
}
private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
// Note the "api", NOT "api-v2"
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
@ -97,18 +107,25 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector;
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
public String getNextPageUrl() throws IOException, ExtractionException {
if(nextPageUrl == null) {
computeStreamsAndNextPageUrl();
}
return nextPageUrl;
}
@Override
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
return new InfoItemPage(collector, nextPageUrl);
}

View File

@ -45,18 +45,18 @@ public class SoundcloudService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return new SoundcloudStreamExtractor(this, url);
}
@Override
public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudChannelExtractor(this, url, nextPageUrl);
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return new SoundcloudChannelExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(this, url, nextPageUrl);
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return new SoundcloudPlaylistExtractor(this, url);
}
@Override
@ -70,12 +70,10 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String nextPageUrl,
String id)
throws ExtractionException, IOException {
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
url,
nextPageUrl,
id);
}
};

View File

@ -18,7 +18,7 @@ import java.util.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public SoundcloudStreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}

View File

@ -50,7 +50,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
/**
* It's lazily initialized (when getInfoItemPage is called)
*/
private Document nextStreamsAjax;
// private Document nextStreamsAjax;
/**
* Unfortunately, we have to fetch the page even if we are only getting next streams,
@ -58,12 +58,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
* <br/>
* This help us to keep track on what are we fetching.
*/
private boolean fetchingNextStreams;
//private boolean fetchingNextStreams;
public YoutubeChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public YoutubeChannelExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
//fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
}
@Override
@ -72,10 +72,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);
if (!fetchingNextStreams) {
nextPageUrl = getNextPageUrlFrom(doc);
}
nextStreamsAjax = null;
//nextStreamsAjax = null;
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(doc);
}
@Nonnull
@ -171,37 +173,38 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams");
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
try {
if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams");
}
fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = JsonParser.object().from(
NewPipe.getDownloader()
.download(pageUrl));
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
collectStreamsFrom(collector, ajaxHtml.select("body").first());
return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse json data for next streams", pe);
}
fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextPageAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return new InfoItemPage(collector, nextPageUrl);
}
private void setupNextPageAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextPageUrl);
try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
} else {
nextPageUrl = "";
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
throws ParsingException {
String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
if (!loadMoreHtmlDataRaw.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));
} else {
return "";
}
}

View File

@ -29,10 +29,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
/**
* It's lazily initialized (when getInfoItemPage is called)
*/
private Document nextStreamsAjax;
private Document nextPageAjax;
public YoutubePlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextPageUrl);
public YoutubePlaylistExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Override
@ -40,8 +40,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
nextPageUrl = getNextPageUrlFrom(doc);
nextStreamsAjax = null;
nextPageAjax = null;
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(doc);
}
@Nonnull
@ -139,34 +143,37 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams");
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
try {
if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
// setupNextStreamsAjax(NewPipe.getDownloader());
final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader()
.download(pageUrl));
final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
+ pageJson.getString("content_html")
+ "</tbody></table>", pageUrl);
collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse ajax json", pe);
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return new InfoItemPage(collector, nextPageUrl);
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextPageUrl);
try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
} else {
nextPageUrl = "";
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
throws ParsingException{
String nextPageHtml = pageJson.getString("load_more_widget_html");
if (!nextPageHtml.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
} else {
return "";
}
}

View File

@ -65,18 +65,18 @@ public class YoutubeService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return new YoutubeStreamExtractor(this, url);
}
@Override
public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubeChannelExtractor(this, url, nextPageUrl);
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return new YoutubeChannelExtractor(this, url);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(this, url, nextPageUrl);
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return new YoutubePlaylistExtractor(this, url);
}
@Override
@ -92,9 +92,9 @@ public class YoutubeService extends StreamingService {
try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextPageUrl, String id)
throws ExtractionException, IOException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, nextPageUrl, id);
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
}
}, new YoutubeTrendingUrlIdHandler(), "Trending");
list.setDefaultKiosk("Trending");

View File

@ -85,7 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public YoutubeStreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}

View File

@ -37,9 +37,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
throws ExtractionException {
super(service, url, nextPageUrl, kioskId);
super(service, url, kioskId);
}
@Override
@ -61,7 +61,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
}
@Override
public ListExtractor.InfoItemPage getInfoItemPage() {
public String getNextPageUrl() {
return "";
}
@Override
public ListExtractor.InfoItemPage getPage(String pageUrl) {
return null;
}

View File

@ -39,13 +39,13 @@ public abstract class StreamExtractor extends Extractor {
public static final int NO_AGE_LIMIT = 0;
public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
public StreamExtractor(StreamingService service, String url) throws ExtractionException {
super(service, url);
}
@Nonnull
@Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException {
protected UrlIdHandler getUrlIdHandler() {
return getService().getStreamUrlIdHandler();
}

View File

@ -73,7 +73,7 @@ public class SoundcloudChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

View File

@ -76,8 +76,8 @@ public class SoundcloudChartsExtractorTest {
@Test
public void testGetNextStreams() throws Exception {
extractor.getInfoItems();
assertFalse("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getInfoItemPage().infoItemList.isEmpty());
assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|| extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty());
}
@Test

View File

@ -90,7 +90,7 @@ public class SoundcloudPlaylistExtractorTest {
extractor.getStreams();
// This playlist don't have more streams, it should throw an error
extractor.getInfoItemPage();
extractor.getPage(extractor.getNextPageUrl());
fail("Expected exception wasn't thrown");
}

View File

@ -112,7 +112,7 @@ public class YoutubeChannelExtractorTest {
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors);
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

View File

@ -111,7 +111,7 @@ public class YoutubePlaylistExtractorTest {
public void testGetNextPage() throws Exception {
// Setup the streams
extractor.getStreams();
ListExtractor.InfoItemPage infoItemPage = extractor.getInfoItemPage();
ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl());
assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors);
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

View File

@ -85,8 +85,8 @@ public class YoutubeTrendingExtractorTest {
@Test
public void testGetNextStreams() throws Exception {
assertTrue("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getInfoItemPage().getNextItemsList().isEmpty());
assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|| extractor.getPage(extractor.getNextPageUrl()).getNextItemsList().isEmpty());
}
@Test