add ability to get donation links from channel

This commit is contained in:
Christian Schabesberger 2018-04-07 16:45:07 +02:00
parent 77a74b8d20
commit 1d94e7d477
5 changed files with 59 additions and 0 deletions

View File

@ -45,4 +45,5 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getFeedUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException;
public abstract long getSubscriberCount() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException;
public abstract String getDescription() throws ParsingException; public abstract String getDescription() throws ParsingException;
public abstract String[] getDonationLinks() throws ParsingException;
} }

View File

@ -92,6 +92,11 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
} catch (Exception e) { } catch (Exception e) {
info.addError(e); info.addError(e);
} }
try {
info.setDonationLinks(extractor.getDonationLinks());
} catch (Exception e) {
info.addError(e);
}
return info; return info;
} }
@ -101,6 +106,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
private String feedUrl; private String feedUrl;
private long subscriberCount = -1; private long subscriberCount = -1;
private String description; private String description;
private String[] donationLinks;
public String getAvatarUrl() { public String getAvatarUrl() {
return avatarUrl; return avatarUrl;
@ -141,4 +147,12 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public void setDescription(String description) { public void setDescription(String description) {
this.description = description; this.description = description;
} }
public String[] getDonationLinks() {
return donationLinks;
}
public void setDonationLinks(String[] donationLinks) {
this.donationLinks = donationLinks;
}
} }

View File

@ -133,4 +133,9 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, nextPageUrl); return new InfoItemsPage<>(collector, nextPageUrl);
} }
@Override
public String[] getDonationLinks() {
return new String[0];
}
} }

View File

@ -4,12 +4,14 @@ package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import com.sun.org.apache.xerces.internal.xs.StringList;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -20,6 +22,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
/* /*
* Created by Christian Schabesberger on 25.07.16. * Created by Christian Schabesberger on 25.07.16.
@ -181,6 +184,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
} }
@Override
public String[] getDonationLinks() throws ParsingException {
try {
ArrayList<String> links = new ArrayList<>();
Element linkHolder = doc.select("div[id=\"links-holder\"]").first();
if(linkHolder == null) {
// this occures if no links are embeded into the channel
return new String[0];
}
for(Element a : linkHolder.select("a")) {
System.err.println(a.attr("abs:href"));
links.add(a.attr("abs:href"));
}
String[] retLinks = new String[links.size()];
retLinks = links.toArray(retLinks);
return retLinks;
} catch (Exception e) {
throw new ParsingException("Could not get donation links", e);
}
}
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
throws ParsingException { throws ParsingException {
String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");

View File

@ -103,6 +103,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception { public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
} }
@Test
public void testChannelDonation() throws Exception {
assertTrue(extractor.getDonationLinks().length != 0);
}
} }
public static class Kurzgesagt implements BaseChannelExtractorTest { public static class Kurzgesagt implements BaseChannelExtractorTest {
@ -205,6 +210,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception { public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6); assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6);
} }
@Test
public void testChannelDonation() throws Exception {
assertTrue(extractor.getDonationLinks().length == 0);
}
} }
public static class CaptainDisillusion implements BaseChannelExtractorTest { public static class CaptainDisillusion implements BaseChannelExtractorTest {
@ -389,6 +399,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception { public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
} }
@Test
public void testChannelDonation() throws Exception {
assertTrue(extractor.getDonationLinks().length == 0);
}
} }
}; };