made getPage() function be stand alone

src/main/java/org/schabi/newpipe/extractor/ListExtractor.java

@@ -10,32 +10,26 @@ import java.util.List;
  * Base class to extractors that have a list (e.g. playlists, users).
  */
 public abstract class ListExtractor extends Extractor {
-    protected String nextPageUrl;
 
     /**
      * Get a new ListExtractor with the given nextPageUrl set.
      */
-    public ListExtractor(StreamingService service, String url, String nextPageUrl) throws ExtractionException {
+    public ListExtractor(StreamingService service, String url) throws ExtractionException {
         super(service, url);
-        setNextPageUrl(nextPageUrl);
     }
 
     @Nonnull
     public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException;
 
-    public abstract InfoItemPage getInfoItemPage() throws IOException, ExtractionException;
+    public abstract String getNextPageUrl() throws IOException, ExtractionException;
 
-    public boolean hasNextPage() {
+    public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException;
-        return nextPageUrl != null && !nextPageUrl.isEmpty();
+
+    public boolean hasNextPage() throws IOException, ExtractionException {
+        return getNextPageUrl() != null && !getNextPageUrl().isEmpty();
     }
 
-    public String getNextPageUrl() {
-        return nextPageUrl;
-    }
 
-    public void setNextPageUrl(String nextPageUrl) {
-        this.nextPageUrl = nextPageUrl;
-    }
 
     /*//////////////////////////////////////////////////////////////////////////
     // Inner

src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -68,20 +68,12 @@ public abstract class StreamingService {
     public abstract UrlIdHandler getPlaylistUrlIdHandler();
     public abstract SearchEngine getSearchEngine();
     public abstract SuggestionExtractor getSuggestionExtractor();
-    public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
+    public abstract StreamExtractor getStreamExtractor(String url) throws ExtractionException;
-    public abstract ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
-    public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
     public abstract KioskList getKioskList() throws ExtractionException;
+    public abstract ChannelExtractor getChannelExtractor(String url) throws ExtractionException;
+    public abstract PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException;
     public abstract SubscriptionExtractor getSubscriptionExtractor();
 
-    public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
-        return getChannelExtractor(url, null);
-    }
-
-    public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
-        return getPlaylistExtractor(url, null);
-    }
-
     /**
      * figure out where the link is pointing to (a channel, video, playlist, etc.)
      */

src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -31,9 +31,9 @@ import java.io.IOException;
 
 public abstract class ChannelExtractor extends ListExtractor {
 
-    public ChannelExtractor(StreamingService service, String url, String nextPageUrl)
+    public ChannelExtractor(StreamingService service, String url)
             throws ExtractionException {
-        super(service, url, nextPageUrl);
+        super(service, url);
     }
 
     @Nonnull
@@ -55,5 +55,4 @@ public abstract class ChannelExtractor extends ListExtractor {
     public abstract String getFeedUrl() throws ParsingException;
     public abstract long getSubscriberCount() throws ParsingException;
     public abstract String getDescription() throws ParsingException;
-
 }

src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -37,9 +37,9 @@ public class ChannelInfo extends ListInfo {
     }
 
 
-    public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl)
+    public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl)
             throws IOException, ExtractionException {
-        return service.getChannelExtractor(url, nextPageUrl).getInfoItemPage();
+        return service.getChannelExtractor(url).getPage(pageUrl);
     }
 
     public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@@ -52,7 +52,7 @@ public class ChannelInfo extends ListInfo {
         return getInfo(extractor);
     }
 
-    public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
+    public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
 
         // important data
         int serviceId = extractor.getServiceId();

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java

@@ -34,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor {
 
     public KioskExtractor(StreamingService streamingService,
                           String url,
-                          String nextPageUrl,
                           String kioskId)
         throws ExtractionException {
-        super(streamingService, url, nextPageUrl);
+        super(streamingService, url);
         this.id = kioskId;
     }
 

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java

@@ -37,12 +37,12 @@ public class KioskInfo extends ListInfo {
 
     public static ListExtractor.InfoItemPage getMoreItems(StreamingService service,
                                                              String url,
-                                                             String nextPageUrl,
+                                                             String pageUrl,
                                                              String contentCountry) throws IOException, ExtractionException {
         KioskList kl = service.getKioskList();
-        KioskExtractor extractor = kl.getExtractorByUrl(url, nextPageUrl);
+        KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
         extractor.setContentCountry(contentCountry);
-        return extractor.getInfoItemPage();
+        return extractor.getPage(pageUrl);
     }
 
     public static KioskInfo getInfo(String url,

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -14,7 +14,6 @@ public  class KioskList {
     public interface KioskExtractorFactory {
         KioskExtractor createNewKiosk(final StreamingService streamingService,
                                              final String url,
-                                             final String nextPageUrl,
                                              final String kioskId)
             throws ExtractionException, IOException;
     }
@@ -74,8 +73,7 @@ public  class KioskList {
             throw new ExtractionException("No kiosk found with the type: " + kioskId);
         } else {
             return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
-                    ke.handler.getUrl(kioskId),
+                    ke.handler.getUrl(kioskId), kioskId);
-                    nextPageUrl, kioskId);
         }
     }
 

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java

@@ -14,8 +14,8 @@ import java.io.IOException;
 
 public abstract class PlaylistExtractor extends ListExtractor {
 
-    public PlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    public PlaylistExtractor(StreamingService service, String url) throws ExtractionException {
-        super(service, url, nextPageUrl);
+        super(service, url);
     }
 
     @Nonnull

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.io.IOException;
 
@@ -17,8 +16,8 @@ public class PlaylistInfo extends ListInfo {
         super(serviceId, id, url, name);
     }
 
-    public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
-        return service.getPlaylistExtractor(url, nextPageUrl).getInfoItemPage();
+        return service.getPlaylistExtractor(url).getPage(pageUrl);
     }
 
     public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
@@ -36,7 +35,7 @@ public class PlaylistInfo extends ListInfo {
      *
      * @param extractor an extractor where fetchPage() was already got called on.
      */
-    public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
+    public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
 
         int serviceId = extractor.getServiceId();
         String url = extractor.getCleanUrl();

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -18,8 +18,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     private String userId;
     private JsonObject user;
 
-    public SoundcloudChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    private StreamInfoItemsCollector streamInfoItemsCollector = null;
-        super(service, url, nextPageUrl);
+    private String nextPageUrl = null;
+
+    public SoundcloudChannelExtractor(StreamingService service, String url) throws ExtractionException {
+        super(service, url);
     }
 
     @Override
@@ -80,32 +83,50 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     }
 
     @Override
-    public String getDescription() throws ParsingException {
+    public String getDescription() {
         return user.getString("description", "");
     }
 
     @Nonnull
     @Override
-    public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
+    public StreamInfoItemsCollector getStreams() throws ExtractionException {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        if(streamInfoItemsCollector == null) {
-
+            computeNextPageAndGetStreams();
-        String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+        }
-                + "?client_id=" + SoundcloudParsingHelper.clientId()
+        return streamInfoItemsCollector;
-                + "&limit=20"
-                + "&linked_partitioning=1";
-
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
-        return collector;
     }
 
     @Override
-    public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
+    public String getNextPageUrl() throws ExtractionException {
+        if(nextPageUrl == null) {
+            computeNextPageAndGetStreams();
+        }
+        return nextPageUrl;
+    }
+
+    private void computeNextPageAndGetStreams() throws ExtractionException {
+        try {
+            streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
+
+            String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+                    + "?client_id=" + SoundcloudParsingHelper.clientId()
+                    + "&limit=20"
+                    + "&linked_partitioning=1";
+
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
+        } catch (Exception e) {
+            throw new ExtractionException("Could not get next page", e);
+        }
+    }
+
+    @Override
+    public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
         if (!hasNextPage()) {
             throw new ExtractionException("Channel doesn't have more streams");
         }
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
 
         return new InfoItemPage(collector, nextPageUrl);
     }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -4,10 +4,12 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import org.schabi.newpipe.extractor.Collector;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
@@ -16,9 +18,12 @@ import javax.annotation.Nonnull;
 public class SoundcloudChartsExtractor extends KioskExtractor {
 	private String url;
 
-    public SoundcloudChartsExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
+	private StreamInfoItemsCollector collector = null;
+	private String nextPageUrl = null;
+
+    public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
             throws ExtractionException {
-        super(service, url, nextPageUrl, kioskId);
+        super(service, url, kioskId);
         this.url = url;
     }
 
@@ -39,21 +44,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
     }
 
     @Override
-    public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
+    public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
         if (!hasNextPage()) {
             throw new ExtractionException("Chart doesn't have more streams");
         }
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl, true);
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
 
         return new InfoItemPage(collector, nextPageUrl);
     }
 
-    @Nonnull
+
-    @Override
+    private void computNextPageAndStreams() throws IOException, ExtractionException {
-    public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
+        collector = new StreamInfoItemsCollector(getServiceId());
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
 
         String apiUrl = "https://api-v2.soundcloud.com/charts" +
                 "?genre=soundcloud:genres:all-music" +
@@ -72,6 +76,22 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
         }
 
         nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+    }
+
+    @Override
+    public String getNextPageUrl() throws IOException, ExtractionException {
+        if(nextPageUrl == null) {
+            computNextPageAndStreams();
+        }
+        return nextPageUrl;
+    }
+
+    @Nonnull
+    @Override
+    public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
+        if(collector == null) {
+            computNextPageAndStreams();
+        }
         return collector;
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -18,8 +18,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     private String playlistId;
     private JsonObject playlist;
 
-    public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    private StreamInfoItemsCollector streamInfoItemsCollector = null;
-        super(service, url, nextPageUrl);
+    private String nextPageUrl = null;
+
+    public SoundcloudPlaylistExtractor(StreamingService service, String url) throws ExtractionException {
+        super(service, url);
     }
 
     @Override
@@ -89,7 +92,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     @Nonnull
     @Override
     public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        if(streamInfoItemsCollector == null) {
+            computeStreamsAndNextPageUrl();
+        }
+        return streamInfoItemsCollector;
+    }
+
+    private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
+        streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
 
         // Note the "api", NOT "api-v2"
         String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
@@ -97,18 +107,25 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
                 + "&limit=20"
                 + "&linked_partitioning=1";
 
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
+        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
-        return collector;
     }
 
     @Override
-    public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
+    public String getNextPageUrl() throws IOException, ExtractionException {
+        if(nextPageUrl == null) {
+            computeStreamsAndNextPageUrl();
+        }
+        return nextPageUrl;
+    }
+
+    @Override
+    public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
         if (!hasNextPage()) {
             throw new ExtractionException("Playlist doesn't have more streams");
         }
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
 
         return new InfoItemPage(collector, nextPageUrl);
     }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -45,18 +45,18 @@ public class SoundcloudService extends StreamingService {
 
 
     @Override
-    public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
+    public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
         return new SoundcloudStreamExtractor(this, url);
     }
 
     @Override
-    public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
+    public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
-        return new SoundcloudChannelExtractor(this, url, nextPageUrl);
+        return new SoundcloudChannelExtractor(this, url);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
+    public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
-        return new SoundcloudPlaylistExtractor(this, url, nextPageUrl);
+        return new SoundcloudPlaylistExtractor(this, url);
     }
 
     @Override
@@ -70,12 +70,10 @@ public class SoundcloudService extends StreamingService {
             @Override
             public KioskExtractor createNewKiosk(StreamingService streamingService,
                                                  String url,
-                                                 String nextPageUrl,
                                                  String id)
-                    throws ExtractionException, IOException {
+                    throws ExtractionException {
                 return new SoundcloudChartsExtractor(SoundcloudService.this,
                         url,
-                        nextPageUrl,
                         id);
             }
         };

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -18,7 +18,7 @@ import java.util.*;
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
 
-    public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+    public SoundcloudStreamExtractor(StreamingService service, String url) throws ExtractionException {
         super(service, url);
     }
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -50,7 +50,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     /**
      * It's lazily initialized (when getInfoItemPage is called)
      */
-    private Document nextStreamsAjax;
+    // private Document nextStreamsAjax;
 
     /**
      * Unfortunately, we have to fetch the page even if we are only getting next streams,
@@ -58,12 +58,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
      * <br/>
      * This help us to keep track on what are we fetching.
      */
-    private boolean fetchingNextStreams;
+    //private boolean fetchingNextStreams;
 
-    public YoutubeChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    public YoutubeChannelExtractor(StreamingService service, String url) throws ExtractionException {
-        super(service, url, nextPageUrl);
+        super(service, url);
 
-        fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
+        //fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
     }
 
     @Override
@@ -72,10 +72,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         String pageContent = downloader.download(channelUrl);
         doc = Jsoup.parse(pageContent, channelUrl);
 
-        if (!fetchingNextStreams) {
+        //nextStreamsAjax = null;
-            nextPageUrl = getNextPageUrlFrom(doc);
+    }
-        }
+
-        nextStreamsAjax = null;
+    @Override
+    public String getNextPageUrl() throws ExtractionException {
+        return getNextPageUrlFrom(doc);
     }
 
     @Nonnull
@@ -171,37 +173,38 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     }
 
     @Override
-    public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
+    public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
-        if (!hasNextPage()) {
+        try {
-            throw new ExtractionException("Channel doesn't have more streams");
+
+            if (!hasNextPage()) {
+                throw new ExtractionException("Channel doesn't have more streams");
+            }
+
+            fetchPage();
+
+            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+            final JsonObject ajaxJson = JsonParser.object().from(
+                    NewPipe.getDownloader()
+                    .download(pageUrl));
+
+            final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
+
+            collectStreamsFrom(collector, ajaxHtml.select("body").first());
+
+            return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
+        } catch (JsonParserException pe) {
+            throw new ParsingException("Could not parse json data for next streams", pe);
         }
-
-        fetchPage();
-
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
-        setupNextPageAjax(NewPipe.getDownloader());
-        collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
-
-        return new InfoItemPage(collector, nextPageUrl);
     }
 
-    private void setupNextPageAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
+    private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
-        String ajaxDataRaw = downloader.download(nextPageUrl);
+        throws ParsingException {
-        try {
+        String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
-            JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
+        if (!loadMoreHtmlDataRaw.isEmpty()) {
-
+            return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));
-            String htmlDataRaw = ajaxData.getString("content_html");
+        } else {
-            nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
+            return "";
-
-            String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
-            if (!nextStreamsHtmlDataRaw.isEmpty()) {
-                nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
-            } else {
-                nextPageUrl = "";
-            }
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
         }
     }
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java

@@ -29,10 +29,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     /**
      * It's lazily initialized (when getInfoItemPage is called)
      */
-    private Document nextStreamsAjax;
+    private Document nextPageAjax;
 
-    public YoutubePlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
+    public YoutubePlaylistExtractor(StreamingService service, String url) throws ExtractionException {
-        super(service, url, nextPageUrl);
+        super(service, url);
     }
 
     @Override
@@ -40,8 +40,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         String pageContent = downloader.download(getCleanUrl());
         doc = Jsoup.parse(pageContent, getCleanUrl());
 
-        nextPageUrl = getNextPageUrlFrom(doc);
+        nextPageAjax = null;
-        nextStreamsAjax = null;
+    }
+
+    @Override
+    public String getNextPageUrl() throws ExtractionException {
+        return getNextPageUrlFrom(doc);
     }
 
     @Nonnull
@@ -139,34 +143,37 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     }
 
     @Override
-    public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
+    public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
-        if (!hasNextPage()) {
+        try {
-            throw new ExtractionException("Playlist doesn't have more streams");
+            if (!hasNextPage()) {
+                throw new ExtractionException("Playlist doesn't have more streams");
+            }
+
+            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            // setupNextStreamsAjax(NewPipe.getDownloader());
+            final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader()
+                    .download(pageUrl));
+            final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
+                    + pageJson.getString("content_html")
+                    + "</tbody></table>", pageUrl);
+
+            collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
+
+
+
+            return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
+        } catch (JsonParserException pe) {
+            throw new ParsingException("Could not parse ajax json", pe);
         }
-
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        setupNextStreamsAjax(NewPipe.getDownloader());
-        collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
-
-        return new InfoItemPage(collector, nextPageUrl);
     }
 
-    private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
+    private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
-        String ajaxDataRaw = downloader.download(nextPageUrl);
+            throws ParsingException{
-        try {
+        String nextPageHtml = pageJson.getString("load_more_widget_html");
-            JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
+        if (!nextPageHtml.isEmpty()) {
-
+            return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
-            String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
+        } else {
-            nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
+            return "";
-
-            String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
-            if (!nextStreamsHtmlDataRaw.isEmpty()) {
-                nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
-            } else {
-                nextPageUrl = "";
-            }
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
         }
     }
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -65,18 +65,18 @@ public class YoutubeService extends StreamingService {
     }
 
     @Override
-    public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
+    public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
         return new YoutubeStreamExtractor(this, url);
     }
 
     @Override
-    public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
+    public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
-        return new YoutubeChannelExtractor(this, url, nextPageUrl);
+        return new YoutubeChannelExtractor(this, url);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
+    public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
-        return new YoutubePlaylistExtractor(this, url, nextPageUrl);
+        return new YoutubePlaylistExtractor(this, url);
     }
 
     @Override
@@ -92,9 +92,9 @@ public class YoutubeService extends StreamingService {
         try {
             list.addKioskEntry(new KioskList.KioskExtractorFactory() {
                 @Override
-                public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextPageUrl, String id)
+                public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
-                throws ExtractionException, IOException {
+                throws ExtractionException {
-                    return new YoutubeTrendingExtractor(YoutubeService.this, url, nextPageUrl, id);
+                    return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
                 }
             }, new YoutubeTrendingUrlIdHandler(), "Trending");
             list.setDefaultKiosk("Trending");

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -85,7 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private boolean isAgeRestricted;
 
-    public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+    public YoutubeStreamExtractor(StreamingService service, String url) throws ExtractionException {
         super(service, url);
     }
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java

@@ -37,9 +37,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
 
     private Document doc;
 
-    public YoutubeTrendingExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
+    public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
             throws ExtractionException {
-        super(service, url, nextPageUrl, kioskId);
+        super(service, url, kioskId);
     }
 
     @Override
@@ -61,7 +61,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
     }
 
     @Override
-    public ListExtractor.InfoItemPage getInfoItemPage() {
+    public String getNextPageUrl() {
+        return "";
+    }
+
+    @Override
+    public ListExtractor.InfoItemPage getPage(String pageUrl) {
         return null;
     }
 

src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -39,13 +39,13 @@ public abstract class StreamExtractor extends Extractor {
 
     public static final int NO_AGE_LIMIT = 0;
 
-    public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+    public StreamExtractor(StreamingService service, String url) throws ExtractionException {
         super(service, url);
     }
 
     @Nonnull
     @Override
-    protected UrlIdHandler getUrlIdHandler() throws ParsingException {
+    protected UrlIdHandler getUrlIdHandler()  {
         return getService().getStreamUrlIdHandler();
     }
 

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java

@@ -73,7 +73,7 @@ public class SoundcloudChannelExtractorTest {
     public void testGetNextStreams() throws Exception {
         // Setup the streams
         extractor.getStreams();
-        ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
+        ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
         assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
         assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
         assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -76,8 +76,8 @@ public class SoundcloudChartsExtractorTest {
     @Test
     public void testGetNextStreams() throws Exception {
         extractor.getInfoItems();
-        assertFalse("extractor has next streams", extractor.getInfoItemPage() == null
+        assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
-                || extractor.getInfoItemPage().infoItemList.isEmpty());
+                || extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty());
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -90,7 +90,7 @@ public class SoundcloudPlaylistExtractorTest {
         extractor.getStreams();
 
         // This playlist don't have more streams, it should throw an error
-        extractor.getInfoItemPage();
+        extractor.getPage(extractor.getNextPageUrl());
 
         fail("Expected exception wasn't thrown");
     }

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -112,7 +112,7 @@ public class YoutubeChannelExtractorTest {
     public void testGetNextStreams() throws Exception {
         // Setup the streams
         extractor.getStreams();
-        ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
+        ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
         assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
         assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors);
         assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -111,7 +111,7 @@ public class YoutubePlaylistExtractorTest {
     public void testGetNextPage() throws Exception {
         // Setup the streams
         extractor.getStreams();
-        ListExtractor.InfoItemPage infoItemPage = extractor.getInfoItemPage();
+        ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl());
         assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty());
         assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors);
         assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java

@@ -85,8 +85,8 @@ public class YoutubeTrendingExtractorTest {
 
     @Test
     public void testGetNextStreams() throws Exception {
-        assertTrue("extractor has next streams", extractor.getInfoItemPage() == null
+        assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
-                || extractor.getInfoItemPage().getNextItemsList().isEmpty());
+                || extractor.getPage(extractor.getNextPageUrl()).getNextItemsList().isEmpty());
     }
 
     @Test