fix fetch page

This commit is contained in:
Schabi 2017-12-29 14:40:42 +01:00
parent c5ce8ec906
commit 044b8fe32f
10 changed files with 24 additions and 33 deletions

View File

@ -15,29 +15,10 @@ public abstract class ListExtractor extends Extractor {
/** /**
* Get a new ListExtractor with the given nextStreamsUrl set. * Get a new ListExtractor with the given nextStreamsUrl set.
* <p>
* The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
* <p>
* You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
* downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
*/ */
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url); super(service, url);
setNextStreamsUrl(nextStreamsUrl); setNextStreamsUrl(nextStreamsUrl);
if (fetchPageUponCreation()) {
fetchPage();
}
}
/**
* Decide if the page will be fetched upon creation.
* <p>
* The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
* don't need or know what is the next page, thus, fetch the page).
*/
protected boolean fetchPageUponCreation() {
return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
} }
@Nonnull @Nonnull

View File

@ -55,6 +55,8 @@ public class ChannelInfo extends ListInfo {
} }
public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
ChannelExtractor extractor = service.getChannelExtractor(url);
extractor.fetchPage();
return getInfo(service.getChannelExtractor(url)); return getInfo(service.getChannelExtractor(url));
} }

View File

@ -65,13 +65,17 @@ public class KioskInfo extends ListInfo {
String contentCountry) throws IOException, ExtractionException { String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList(); KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null); KioskExtractor extractor = kl.getExtractorByUrl(url, null);
return getInfo(extractor, contentCountry);
}
public static KioskInfo getInfo(KioskExtractor extractor,
String contentCountry) throws IOException, ExtractionException {
extractor.setContentCountry(contentCountry); extractor.setContentCountry(contentCountry);
extractor.fetchPage(); extractor.fetchPage();
return getInfo(extractor);
}
/**
* Get KioskInfo from KioskExtractor
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static KioskInfo getInfo(KioskExtractor extractor) throws ExtractionException {
int serviceId = extractor.getServiceId(); int serviceId = extractor.getServiceId();
String name = extractor.getName(); String name = extractor.getName();

View File

@ -32,9 +32,16 @@ public class PlaylistInfo extends ListInfo {
} }
public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getPlaylistExtractor(url)); PlaylistExtractor extractor = service.getPlaylistExtractor(url);
extractor.fetchPage();
return getInfo(extractor);
} }
/**
* Get PlaylistInfo from PlaylistExtractor
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException { public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
int serviceId = extractor.getServiceId(); int serviceId = extractor.getServiceId();

View File

@ -70,14 +70,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
nextStreamsAjax = null; nextStreamsAjax = null;
} }
@Override
protected boolean fetchPageUponCreation() {
// Unfortunately, we have to fetch the page even if we are getting only next streams,
// as they don't deliver enough information on their own (the channel name, for example).
fetchingNextStreams = nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
return true;
}
@Nonnull @Nonnull
@Override @Override
public String getCleanUrl() { public String getCleanUrl() {

View File

@ -23,6 +23,7 @@ public class SoundcloudChannelExtractorTest {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = SoundCloud.getService() extractor = SoundCloud.getService()
.getChannelExtractor("https://soundcloud.com/liluzivert"); .getChannelExtractor("https://soundcloud.com/liluzivert");
extractor.fetchPage();
} }
@Test @Test

View File

@ -22,6 +22,7 @@ public class SoundcloudPlaylistExtractorTest {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = SoundCloud.getService() extractor = SoundCloud.getService()
.getPlaylistExtractor("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r"); .getPlaylistExtractor("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r");
extractor.fetchPage();
} }
@Test @Test

View File

@ -44,6 +44,7 @@ public class YoutubeChannelExtractorTest {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = (YoutubeChannelExtractor) YouTube.getService() extractor = (YoutubeChannelExtractor) YouTube.getService()
.getChannelExtractor("https://www.youtube.com/user/Gronkh"); .getChannelExtractor("https://www.youtube.com/user/Gronkh");
extractor.fetchPage();
} }
@Test @Test

View File

@ -30,6 +30,7 @@ public class YoutubePlaylistExtractorTest {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube.getService() extractor = (YoutubePlaylistExtractor) YouTube.getService()
.getPlaylistExtractor("https://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"); .getPlaylistExtractor("https://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
extractor.fetchPage();
} }
@Test @Test

View File

@ -48,6 +48,7 @@ public class YoutubeTrendingExtractorTest {
extractor = (YoutubeTrendingExtractor) YouTube.getService() extractor = (YoutubeTrendingExtractor) YouTube.getService()
.getKioskList() .getKioskList()
.getExtractorById("Trending", null); .getExtractorById("Trending", null);
extractor.fetchPage();
} }
@Test @Test