diff --git a/extractor/build.gradle b/extractor/build.gradle index 33498cbe8..1b7fbf001 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -5,6 +5,7 @@ dependencies { implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' + implementation 'org.nibor.autolink:autolink:0.8.0' testImplementation 'junit:junit:4.12' } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index 217056bf7..e39e6977a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -35,7 +35,8 @@ public enum MediaFormat { // audio formats M4A (0x3, "m4a", "m4a", "audio/mp4"), WEBMA (0x4, "WebM", "webm", "audio/webm"), - MP3 (0x5, "MP3", "mp3", "audio/mpeg"); + MP3 (0x5, "MP3", "mp3", "audio/mpeg"), + OPUS (0x6, "opus", "opus", "audio/opus"); public final int id; public final String name; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 6a920ad94..05a5bc587 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -45,4 +45,5 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; + public abstract String[] getDonationLinks() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 1691f9544..0d8a860bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -92,6 +92,11 @@ public class ChannelInfo extends ListInfo { } catch (Exception e) { info.addError(e); } + try { + info.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + info.addError(e); + } return info; } @@ -101,6 +106,7 @@ public class ChannelInfo extends ListInfo { private String feedUrl; private long subscriberCount = -1; private String description; + private String[] donationLinks; public String getAvatarUrl() { return avatarUrl; @@ -141,4 +147,12 @@ public class ChannelInfo extends ListInfo { public void setDescription(String description) { this.description = description; } + + public String[] getDonationLinks() { + return donationLinks; + } + + public void setDonationLinks(String[] donationLinks) { + this.donationLinks = donationLinks; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 0df15fa69..7232ac5ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -133,4 +133,9 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, nextPageUrl); } + + @Override + public String[] getDonationLinks() { + return new String[0]; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index dddc4d5c4..8de597d88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -209,6 +209,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return collector; } + @Override + public String[] getDonationLinks() { + return new String[0]; + } @Override public String getErrorMessage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index de0cb625f..195d20abc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -4,22 +4,26 @@ package org.schabi.newpipe.extractor.services.youtube; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.sun.org.apache.xerces.internal.xs.StringList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.ArrayList; /* * Created by Christian Schabesberger on 25.07.16. @@ -181,6 +185,29 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); } + @Override + public String[] getDonationLinks() throws ParsingException { + try { + ArrayList links = new ArrayList<>(); + Element linkHolder = doc.select("div[id=\"header-links\"]").first(); + if(linkHolder == null) { + // this occures if no links are embeded into the channel + return new String[0]; + } + for(Element a : linkHolder.select("a")) { + String link = a.attr("abs:href"); + if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { + links.add(link); + } + } + String[] retLinks = new String[links.size()]; + retLinks = links.toArray(retLinks); + return retLinks; + } catch (Exception e) { + throw new ParsingException("Could not get donation links", e); + } + } + private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) throws ParsingException { String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index be6d96c7a..c9839ac69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -528,6 +529,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { return errorReason != null ? errorReason.toString() : null; } + @Override + public String[] getDonationLinks() throws ParsingException { + try { + ArrayList donationLinks = new ArrayList<>(); + for (String s : Parser.getLinksFromString(getDescription())) { + if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { + donationLinks.add(s); + } + } + String[] donlret = new String[donationLinks.size()]; + donlret = donationLinks.toArray(donlret); + return donlret; + } catch (Exception e) { + throw new ParsingException("Could not get donation links", e); + } + } + + /*////////////////////////////////////////////////////////////////////////// // Fetch page //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a3df7ecfa..0aeedd029 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -141,6 +141,8 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; + public abstract String[] getDonationLinks() throws ExtractionException; + /** * Analyses the webpage's document and extracts any error message there might be. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java new file mode 100644 index 000000000..b62e062f3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor.utils; + +import java.net.MalformedURLException; +import java.net.URL; + +public class DonationLinkHelper { + public enum DonationService { + NO_DONATION, + PATREON, + PAYPAL + } + + + public static DonationService getServiceByLink(String link) throws MalformedURLException { + URL url = new URL(link); + switch (url.getHost()) { + case "www.patreon.com": + return DonationService.PATREON; + case "patreon.com": + return DonationService.PATREON; + case "paypal.me": + return DonationService.PAYPAL; + case "www.paypal.me": + return DonationService.PAYPAL; + default: + return DonationService.NO_DONATION; + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 9920c7fb4..42f098dce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -1,9 +1,14 @@ package org.schabi.newpipe.extractor.utils; +import org.nibor.autolink.LinkExtractor; +import org.nibor.autolink.LinkSpan; +import org.nibor.autolink.LinkType; import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -80,4 +85,23 @@ public class Parser { } return map; } + + public static String[] getLinksFromString(final String txt) throws ParsingException { + try { + ArrayList links = new ArrayList<>(); + LinkExtractor linkExtractor = LinkExtractor.builder() + .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) + .build(); + Iterable linkss = linkExtractor.extractLinks(txt); + for(LinkSpan ls : linkss) { + links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); + } + + String[] linksarray = new String[links.size()]; + linksarray = links.toArray(linksarray); + return linksarray; + } catch (Exception e) { + throw new ParsingException("Could not get links from string", e); + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 8fb520773..510f210a9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; @@ -103,6 +104,13 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); } + + @Test + public void testChannelDonation() throws Exception { + // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface + // in order to make this work + assertTrue(extractor.getDonationLinks().length == 0); + } } public static class Kurzgesagt implements BaseChannelExtractorTest { @@ -205,6 +213,11 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6); } + + @Test + public void testChannelDonation() throws Exception { + assertTrue(extractor.getDonationLinks().length == 1); + } } public static class CaptainDisillusion implements BaseChannelExtractorTest { @@ -389,6 +402,11 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); } + + @Test + public void testChannelDonation() throws Exception { + assertTrue(extractor.getDonationLinks().length == 0); + } } }; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java new file mode 100644 index 000000000..8bc82ded8 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.services.youtube; + + +/* + * Created by Christian Schabesberger on 30.12.15. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeStreamExtractorDonationTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.stream.StreamExtractor; + +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +/** + * Test for {@link StreamExtractor} + */ +public class YoutubeStreamExtractorDonationTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=pXb3jERMoI0"); + extractor.fetchPage(); + } + + @Test + public void getDonationLinksTest() throws Exception { + assertTrue(String.valueOf(extractor.getDonationLinks().length), + extractor.getDonationLinks().length == 2); + } +}