From d812a114c52938e8f9f010100815ec7ed31394f1 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 26 Feb 2018 15:55:27 +0100 Subject: [PATCH] made getPage() function be stand alone --- .../newpipe/extractor/ListExtractor.java | 18 ++--- .../newpipe/extractor/StreamingService.java | 14 +--- .../extractor/channel/ChannelExtractor.java | 5 +- .../extractor/channel/ChannelInfo.java | 6 +- .../extractor/kiosk/KioskExtractor.java | 3 +- .../newpipe/extractor/kiosk/KioskInfo.java | 6 +- .../newpipe/extractor/kiosk/KioskList.java | 4 +- .../extractor/playlist/PlaylistExtractor.java | 4 +- .../extractor/playlist/PlaylistInfo.java | 7 +- .../SoundcloudChannelExtractor.java | 51 ++++++++---- .../soundcloud/SoundcloudChartsExtractor.java | 36 +++++++-- .../SoundcloudPlaylistExtractor.java | 31 ++++++-- .../soundcloud/SoundcloudService.java | 14 ++-- .../soundcloud/SoundcloudStreamExtractor.java | 2 +- .../youtube/YoutubeChannelExtractor.java | 77 ++++++++++--------- .../youtube/YoutubePlaylistExtractor.java | 67 ++++++++-------- .../services/youtube/YoutubeService.java | 16 ++-- .../youtube/YoutubeStreamExtractor.java | 2 +- .../youtube/YoutubeTrendingExtractor.java | 11 ++- .../extractor/stream/StreamExtractor.java | 4 +- .../SoundcloudChannelExtractorTest.java | 2 +- .../SoundcloudChartsExtractorTest.java | 4 +- .../SoundcloudPlaylistExtractorTest.java | 2 +- .../youtube/YoutubeChannelExtractorTest.java | 2 +- .../youtube/YoutubePlaylistExtractorTest.java | 2 +- .../youtube/YoutubeTrendingExtractorTest.java | 4 +- 26 files changed, 223 insertions(+), 171 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index aeb1463de..746ab7867 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -10,32 +10,26 @@ import java.util.List; * Base class to extractors that have a list (e.g. playlists, users). */ public abstract class ListExtractor extends Extractor { - protected String nextPageUrl; /** * Get a new ListExtractor with the given nextPageUrl set. */ - public ListExtractor(StreamingService service, String url, String nextPageUrl) throws ExtractionException { + public ListExtractor(StreamingService service, String url) throws ExtractionException { super(service, url); - setNextPageUrl(nextPageUrl); } @Nonnull public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException; - public abstract InfoItemPage getInfoItemPage() throws IOException, ExtractionException; + public abstract String getNextPageUrl() throws IOException, ExtractionException; - public boolean hasNextPage() { - return nextPageUrl != null && !nextPageUrl.isEmpty(); + public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException; + + public boolean hasNextPage() throws IOException, ExtractionException { + return getNextPageUrl() != null && !getNextPageUrl().isEmpty(); } - public String getNextPageUrl() { - return nextPageUrl; - } - public void setNextPageUrl(String nextPageUrl) { - this.nextPageUrl = nextPageUrl; - } /*////////////////////////////////////////////////////////////////////////// // Inner diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 9ae75168b..969e5f089 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -68,20 +68,12 @@ public abstract class StreamingService { public abstract UrlIdHandler getPlaylistUrlIdHandler(); public abstract SearchEngine getSearchEngine(); public abstract SuggestionExtractor getSuggestionExtractor(); - public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; - public abstract ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException; - public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException; + public abstract StreamExtractor getStreamExtractor(String url) throws ExtractionException; public abstract KioskList getKioskList() throws ExtractionException; + public abstract ChannelExtractor getChannelExtractor(String url) throws ExtractionException; + public abstract PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException; public abstract SubscriptionExtractor getSubscriptionExtractor(); - public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { - return getChannelExtractor(url, null); - } - - public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException { - return getPlaylistExtractor(url, null); - } - /** * figure out where the link is pointing to (a channel, video, playlist, etc.) */ diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index efd24c93b..947c77603 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -31,9 +31,9 @@ import java.io.IOException; public abstract class ChannelExtractor extends ListExtractor { - public ChannelExtractor(StreamingService service, String url, String nextPageUrl) + public ChannelExtractor(StreamingService service, String url) throws ExtractionException { - super(service, url, nextPageUrl); + super(service, url); } @Nonnull @@ -55,5 +55,4 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; - } diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 55c67490a..0253ca99a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -37,9 +37,9 @@ public class ChannelInfo extends ListInfo { } - public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) + public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { - return service.getChannelExtractor(url, nextPageUrl).getInfoItemPage(); + return service.getChannelExtractor(url).getPage(pageUrl); } public static ChannelInfo getInfo(String url) throws IOException, ExtractionException { @@ -52,7 +52,7 @@ public class ChannelInfo extends ListInfo { return getInfo(extractor); } - public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException { + public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException { // important data int serviceId = extractor.getServiceId(); diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 5b6b7eeab..1ad59094e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -34,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor { public KioskExtractor(StreamingService streamingService, String url, - String nextPageUrl, String kioskId) throws ExtractionException { - super(streamingService, url, nextPageUrl); + super(streamingService, url); this.id = kioskId; } diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 36b0e3b8f..a666590b7 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -37,12 +37,12 @@ public class KioskInfo extends ListInfo { public static ListExtractor.InfoItemPage getMoreItems(StreamingService service, String url, - String nextPageUrl, + String pageUrl, String contentCountry) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); - KioskExtractor extractor = kl.getExtractorByUrl(url, nextPageUrl); + KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); extractor.setContentCountry(contentCountry); - return extractor.getInfoItemPage(); + return extractor.getPage(pageUrl); } public static KioskInfo getInfo(String url, diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 6120b0773..fb8e48680 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -14,7 +14,6 @@ public class KioskList { public interface KioskExtractorFactory { KioskExtractor createNewKiosk(final StreamingService streamingService, final String url, - final String nextPageUrl, final String kioskId) throws ExtractionException, IOException; } @@ -74,8 +73,7 @@ public class KioskList { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id), - ke.handler.getUrl(kioskId), - nextPageUrl, kioskId); + ke.handler.getUrl(kioskId), kioskId); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 9e79cf367..a627310c0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -14,8 +14,8 @@ import java.io.IOException; public abstract class PlaylistExtractor extends ListExtractor { - public PlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - super(service, url, nextPageUrl); + public PlaylistExtractor(StreamingService service, String url) throws ExtractionException { + super(service, url); } @Nonnull diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 9f213eeb4..d05c46b2e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.IOException; @@ -17,8 +16,8 @@ public class PlaylistInfo extends ListInfo { super(serviceId, id, url, name); } - public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - return service.getPlaylistExtractor(url, nextPageUrl).getInfoItemPage(); + public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { + return service.getPlaylistExtractor(url).getPage(pageUrl); } public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException { @@ -36,7 +35,7 @@ public class PlaylistInfo extends ListInfo { * * @param extractor an extractor where fetchPage() was already got called on. */ - public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException { + public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException { int serviceId = extractor.getServiceId(); String url = extractor.getCleanUrl(); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index bb66df351..9795c90e4 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -18,8 +18,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; private JsonObject user; - public SoundcloudChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - super(service, url, nextPageUrl); + private StreamInfoItemsCollector streamInfoItemsCollector = null; + private String nextPageUrl = null; + + public SoundcloudChannelExtractor(StreamingService service, String url) throws ExtractionException { + super(service, url); } @Override @@ -80,32 +83,50 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public String getDescription() throws ParsingException { + public String getDescription() { return user.getString("description", ""); } @Nonnull @Override - public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=20" - + "&linked_partitioning=1"; - - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); - return collector; + public StreamInfoItemsCollector getStreams() throws ExtractionException { + if(streamInfoItemsCollector == null) { + computeNextPageAndGetStreams(); + } + return streamInfoItemsCollector; } @Override - public InfoItemPage getInfoItemPage() throws IOException, ExtractionException { + public String getNextPageUrl() throws ExtractionException { + if(nextPageUrl == null) { + computeNextPageAndGetStreams(); + } + return nextPageUrl; + } + + private void computeNextPageAndGetStreams() throws ExtractionException { + try { + streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + + String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks" + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=20" + + "&linked_partitioning=1"; + + nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); + } catch (Exception e) { + throw new ExtractionException("Could not get next page", e); + } + } + + @Override + public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { if (!hasNextPage()) { throw new ExtractionException("Channel doesn't have more streams"); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl); + String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); return new InfoItemPage(collector, nextPageUrl); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 0f62ae87e..0bef85d89 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.schabi.newpipe.extractor.Collector; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -16,9 +18,12 @@ import javax.annotation.Nonnull; public class SoundcloudChartsExtractor extends KioskExtractor { private String url; - public SoundcloudChartsExtractor(StreamingService service, String url, String nextPageUrl, String kioskId) + private StreamInfoItemsCollector collector = null; + private String nextPageUrl = null; + + public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId) throws ExtractionException { - super(service, url, nextPageUrl, kioskId); + super(service, url, kioskId); this.url = url; } @@ -39,21 +44,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public InfoItemPage getInfoItemPage() throws IOException, ExtractionException { + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { if (!hasNextPage()) { throw new ExtractionException("Chart doesn't have more streams"); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl, true); + String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true); return new InfoItemPage(collector, nextPageUrl); } - @Nonnull - @Override - public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + private void computNextPageAndStreams() throws IOException, ExtractionException { + collector = new StreamInfoItemsCollector(getServiceId()); String apiUrl = "https://api-v2.soundcloud.com/charts" + "?genre=soundcloud:genres:all-music" + @@ -72,6 +76,22 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + if(nextPageUrl == null) { + computNextPageAndStreams(); + } + return nextPageUrl; + } + + @Nonnull + @Override + public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { + if(collector == null) { + computNextPageAndStreams(); + } return collector; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 9216ebaf4..8ab6c6d07 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -18,8 +18,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private String playlistId; private JsonObject playlist; - public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - super(service, url, nextPageUrl); + private StreamInfoItemsCollector streamInfoItemsCollector = null; + private String nextPageUrl = null; + + public SoundcloudPlaylistExtractor(StreamingService service, String url) throws ExtractionException { + super(service, url); } @Override @@ -89,7 +92,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(streamInfoItemsCollector == null) { + computeStreamsAndNextPageUrl(); + } + return streamInfoItemsCollector; + } + + private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException { + streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); // Note the "api", NOT "api-v2" String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks" @@ -97,18 +107,25 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + "&limit=20" + "&linked_partitioning=1"; - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); - return collector; + nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); } @Override - public InfoItemPage getInfoItemPage() throws IOException, ExtractionException { + public String getNextPageUrl() throws IOException, ExtractionException { + if(nextPageUrl == null) { + computeStreamsAndNextPageUrl(); + } + return nextPageUrl; + } + + @Override + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { if (!hasNextPage()) { throw new ExtractionException("Playlist doesn't have more streams"); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl); + String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); return new InfoItemPage(collector, nextPageUrl); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 636e55338..58d2171c2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -45,18 +45,18 @@ public class SoundcloudService extends StreamingService { @Override - public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException { + public StreamExtractor getStreamExtractor(String url) throws ExtractionException { return new SoundcloudStreamExtractor(this, url); } @Override - public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException { - return new SoundcloudChannelExtractor(this, url, nextPageUrl); + public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { + return new SoundcloudChannelExtractor(this, url); } @Override - public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException { - return new SoundcloudPlaylistExtractor(this, url, nextPageUrl); + public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { + return new SoundcloudPlaylistExtractor(this, url); } @Override @@ -70,12 +70,10 @@ public class SoundcloudService extends StreamingService { @Override public KioskExtractor createNewKiosk(StreamingService streamingService, String url, - String nextPageUrl, String id) - throws ExtractionException, IOException { + throws ExtractionException { return new SoundcloudChartsExtractor(SoundcloudService.this, url, - nextPageUrl, id); } }; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 6a14f3b13..2abececd3 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -18,7 +18,7 @@ import java.util.*; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; - public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + public SoundcloudStreamExtractor(StreamingService service, String url) throws ExtractionException { super(service, url); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index b24db213e..00a376aa0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -50,7 +50,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { /** * It's lazily initialized (when getInfoItemPage is called) */ - private Document nextStreamsAjax; + // private Document nextStreamsAjax; /** * Unfortunately, we have to fetch the page even if we are only getting next streams, @@ -58,12 +58,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { *
* This help us to keep track on what are we fetching. */ - private boolean fetchingNextStreams; + //private boolean fetchingNextStreams; - public YoutubeChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - super(service, url, nextPageUrl); + public YoutubeChannelExtractor(StreamingService service, String url) throws ExtractionException { + super(service, url); - fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty(); + //fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty(); } @Override @@ -72,10 +72,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); - if (!fetchingNextStreams) { - nextPageUrl = getNextPageUrlFrom(doc); - } - nextStreamsAjax = null; + //nextStreamsAjax = null; + } + + @Override + public String getNextPageUrl() throws ExtractionException { + return getNextPageUrlFrom(doc); } @Nonnull @@ -171,37 +173,38 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemPage getInfoItemPage() throws IOException, ExtractionException { - if (!hasNextPage()) { - throw new ExtractionException("Channel doesn't have more streams"); + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + try { + + if (!hasNextPage()) { + throw new ExtractionException("Channel doesn't have more streams"); + } + + fetchPage(); + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + final JsonObject ajaxJson = JsonParser.object().from( + NewPipe.getDownloader() + .download(pageUrl)); + + final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html")); + + collectStreamsFrom(collector, ajaxHtml.select("body").first()); + + return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); + } catch (JsonParserException pe) { + throw new ParsingException("Could not parse json data for next streams", pe); } - - fetchPage(); - - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - setupNextPageAjax(NewPipe.getDownloader()); - collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); - - return new InfoItemPage(collector, nextPageUrl); } - private void setupNextPageAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { - String ajaxDataRaw = downloader.download(nextPageUrl); - try { - JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); - - String htmlDataRaw = ajaxData.getString("content_html"); - nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl); - - String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); - if (!nextStreamsHtmlDataRaw.isEmpty()) { - nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl)); - } else { - nextPageUrl = ""; - } - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); + private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) + throws ParsingException { + String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); + if (!loadMoreHtmlDataRaw.isEmpty()) { + return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl)); + } else { + return ""; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index a92dc3aad..d140c64b4 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -29,10 +29,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { /** * It's lazily initialized (when getInfoItemPage is called) */ - private Document nextStreamsAjax; + private Document nextPageAjax; - public YoutubePlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException { - super(service, url, nextPageUrl); + public YoutubePlaylistExtractor(StreamingService service, String url) throws ExtractionException { + super(service, url); } @Override @@ -40,8 +40,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { String pageContent = downloader.download(getCleanUrl()); doc = Jsoup.parse(pageContent, getCleanUrl()); - nextPageUrl = getNextPageUrlFrom(doc); - nextStreamsAjax = null; + nextPageAjax = null; + } + + @Override + public String getNextPageUrl() throws ExtractionException { + return getNextPageUrlFrom(doc); } @Nonnull @@ -139,34 +143,37 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public InfoItemPage getInfoItemPage() throws IOException, ExtractionException { - if (!hasNextPage()) { - throw new ExtractionException("Playlist doesn't have more streams"); + public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + try { + if (!hasNextPage()) { + throw new ExtractionException("Playlist doesn't have more streams"); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + // setupNextStreamsAjax(NewPipe.getDownloader()); + final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader() + .download(pageUrl)); + final Document pageHtml = Jsoup.parse("" + + pageJson.getString("content_html") + + "
", pageUrl); + + collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); + + + + return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); + } catch (JsonParserException pe) { + throw new ParsingException("Could not parse ajax json", pe); } - - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - setupNextStreamsAjax(NewPipe.getDownloader()); - collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); - - return new InfoItemPage(collector, nextPageUrl); } - private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { - String ajaxDataRaw = downloader.download(nextPageUrl); - try { - JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); - - String htmlDataRaw = "" + ajaxData.getString("content_html") + "
"; - nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl); - - String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); - if (!nextStreamsHtmlDataRaw.isEmpty()) { - nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl)); - } else { - nextPageUrl = ""; - } - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); + private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl) + throws ParsingException{ + String nextPageHtml = pageJson.getString("load_more_widget_html"); + if (!nextPageHtml.isEmpty()) { + return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl)); + } else { + return ""; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index d52dbda71..714c80be2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -65,18 +65,18 @@ public class YoutubeService extends StreamingService { } @Override - public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException { + public StreamExtractor getStreamExtractor(String url) throws ExtractionException { return new YoutubeStreamExtractor(this, url); } @Override - public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException { - return new YoutubeChannelExtractor(this, url, nextPageUrl); + public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { + return new YoutubeChannelExtractor(this, url); } @Override - public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException { - return new YoutubePlaylistExtractor(this, url, nextPageUrl); + public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { + return new YoutubePlaylistExtractor(this, url); } @Override @@ -92,9 +92,9 @@ public class YoutubeService extends StreamingService { try { list.addKioskEntry(new KioskList.KioskExtractorFactory() { @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextPageUrl, String id) - throws ExtractionException, IOException { - return new YoutubeTrendingExtractor(YoutubeService.this, url, nextPageUrl, id); + public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) + throws ExtractionException { + return new YoutubeTrendingExtractor(YoutubeService.this, url, id); } }, new YoutubeTrendingUrlIdHandler(), "Trending"); list.setDefaultKiosk("Trending"); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index c5e4c5f95..0fde3cf19 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -85,7 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + public YoutubeStreamExtractor(StreamingService service, String url) throws ExtractionException { super(service, url); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 954557265..3628cdfd7 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -37,9 +37,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; - public YoutubeTrendingExtractor(StreamingService service, String url, String nextPageUrl, String kioskId) + public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId) throws ExtractionException { - super(service, url, nextPageUrl, kioskId); + super(service, url, kioskId); } @Override @@ -61,7 +61,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public ListExtractor.InfoItemPage getInfoItemPage() { + public String getNextPageUrl() { + return ""; + } + + @Override + public ListExtractor.InfoItemPage getPage(String pageUrl) { return null; } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index b54874afb..b95cff374 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -39,13 +39,13 @@ public abstract class StreamExtractor extends Extractor { public static final int NO_AGE_LIMIT = 0; - public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + public StreamExtractor(StreamingService service, String url) throws ExtractionException { super(service, url); } @Nonnull @Override - protected UrlIdHandler getUrlIdHandler() throws ParsingException { + protected UrlIdHandler getUrlIdHandler() { return getService().getStreamUrlIdHandler(); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index bcf7dd471..560b08061 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -73,7 +73,7 @@ public class SoundcloudChannelExtractorTest { public void testGetNextStreams() throws Exception { // Setup the streams extractor.getStreams(); - ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage(); + ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty()); assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index c6446eec0..dba57aeb6 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -76,8 +76,8 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetNextStreams() throws Exception { extractor.getInfoItems(); - assertFalse("extractor has next streams", extractor.getInfoItemPage() == null - || extractor.getInfoItemPage().infoItemList.isEmpty()); + assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null + || extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 3a3277784..600b0d052 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -90,7 +90,7 @@ public class SoundcloudPlaylistExtractorTest { extractor.getStreams(); // This playlist don't have more streams, it should throw an error - extractor.getInfoItemPage(); + extractor.getPage(extractor.getNextPageUrl()); fail("Expected exception wasn't thrown"); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 948007283..2cb0fcffe 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -112,7 +112,7 @@ public class YoutubeChannelExtractorTest { public void testGetNextStreams() throws Exception { // Setup the streams extractor.getStreams(); - ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage(); + ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty()); assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index e22ca32b3..bbd5dccc0 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -111,7 +111,7 @@ public class YoutubePlaylistExtractorTest { public void testGetNextPage() throws Exception { // Setup the streams extractor.getStreams(); - ListExtractor.InfoItemPage infoItemPage = extractor.getInfoItemPage(); + ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl()); assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty()); assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index f372f283f..dc032b5c0 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -85,8 +85,8 @@ public class YoutubeTrendingExtractorTest { @Test public void testGetNextStreams() throws Exception { - assertTrue("extractor has next streams", extractor.getInfoItemPage() == null - || extractor.getInfoItemPage().getNextItemsList().isEmpty()); + assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null + || extractor.getPage(extractor.getNextPageUrl()).getNextItemsList().isEmpty()); } @Test