be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

fix yt trending contains to less items

This commit is contained in:
Christian Schabesberger 2018-01-04 04:46:27 +01:00
parent a0e5c88b13
commit 5f2d0cf6b5
2 changed files with 71 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
View File

@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -37,7 +38,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc; private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId) public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId)
throws IOException, ExtractionException { throws IOException, ExtractionException {
super(service, url, nextStreamsUrl, kioskId); super(service, url, nextStreamsUrl, kioskId);
} }
@ -81,76 +82,78 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
@Override @Override
public StreamInfoItemCollector getStreams() throws ParsingException { public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first(); Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
for(final Element li : ul.children()) { for(Element ul : uls) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); for(final Element li : ul.children()) {
collector.commit(new YoutubeStreamInfoItemExtractor(li) { final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
@Override collector.commit(new YoutubeStreamInfoItemExtractor(li) {
public String getUrl() throws ParsingException { @Override
try { public String getUrl() throws ParsingException {
Element dl = el.select("h3").first().select("a").first(); try {
return dl.attr("abs:href"); Element dl = el.select("h3").first().select("a").first();
} catch (Exception e) { return dl.attr("abs:href");
throw new ParsingException("Could not get web page url for the video", e); } catch (Exception e) {
} throw new ParsingException("Could not get web page url for the video", e);
}
@Override
public String getName() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getUploaderUrl() throws ParsingException {
try {
String link = getUploaderLink().attr("href");
if(link.isEmpty()) {
throw new IllegalArgumentException("is empty");
} }
return link;
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
} }
}
private Element getUploaderLink() { @Override
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); public String getName() throws ParsingException {
return uploaderEl.select("a").first(); try {
} Element dl = el.select("h3").first().select("a").first();
return dl.text();
@Override } catch (Exception e) {
public String getUploaderName() throws ParsingException { throw new ParsingException("Could not get web page url for the video", e);
try {
return getUploaderLink().text();
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
} }
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
} }
}
}); @Override
public String getUploaderUrl() throws ParsingException {
try {
String link = getUploaderLink().attr("href");
if (link.isEmpty()) {
throw new IllegalArgumentException("is empty");
}
return link;
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
private Element getUploaderLink() {
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
return uploaderEl.select("a").first();
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getUploaderLink().text();
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
});
}
} }
return collector; return collector;

4
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -67,10 +67,10 @@ public class YoutubeTrendingExtractorTest {
} }
@Test @Test
public void testGetStreams() throws Exception { public void testGetStreamsQuantity() throws Exception {
StreamInfoItemCollector collector = extractor.getStreams(); StreamInfoItemCollector collector = extractor.getStreams();
Utils.printErrors(collector); Utils.printErrors(collector);
assertFalse("no streams are received", collector.getItemList().isEmpty()); assertTrue("no streams are received", collector.getItemList().size() >= 20);
} }
@Test @Test