Merge pull request #78 from TeamNewPipe/donation

enable Donations for streams

extractor/build.gradle

@@ -5,6 +5,7 @@ dependencies {
     implementation 'org.jsoup:jsoup:1.9.2'
     implementation 'org.mozilla:rhino:1.7.7.1'
     implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
+    implementation 'org.nibor.autolink:autolink:0.8.0'
 
     testImplementation 'junit:junit:4.12'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java

@@ -35,7 +35,8 @@ public enum MediaFormat {
     // audio formats
     M4A         (0x3,   "m4a",   "m4a",  "audio/mp4"),
     WEBMA       (0x4,   "WebM",  "webm", "audio/webm"),
-    MP3         (0x5,   "MP3",   "mp3",  "audio/mpeg");
+    MP3         (0x5,   "MP3",   "mp3",  "audio/mpeg"),
+    OPUS        (0x6,   "opus",  "opus", "audio/opus");
 
     public final int id;
     public final String name;

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -45,4 +45,5 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
     public abstract String getFeedUrl() throws ParsingException;
     public abstract long getSubscriberCount() throws ParsingException;
     public abstract String getDescription() throws ParsingException;
+    public abstract String[] getDonationLinks() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -92,6 +92,11 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
         } catch (Exception e) {
             info.addError(e);
         }
+        try {
+            info.setDonationLinks(extractor.getDonationLinks());
+        } catch (Exception e) {
+            info.addError(e);
+        }
 
         return info;
     }
@@ -101,6 +106,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
     private String feedUrl;
     private long subscriberCount = -1;
     private String description;
+    private String[] donationLinks;
 
     public String getAvatarUrl() {
         return avatarUrl;
@@ -141,4 +147,12 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
     public void setDescription(String description) {
         this.description = description;
     }
+
+    public String[] getDonationLinks() {
+        return donationLinks;
+    }
+
+    public void setDonationLinks(String[] donationLinks) {
+        this.donationLinks = donationLinks;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -133,4 +133,9 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
 
         return new InfoItemsPage<>(collector, nextPageUrl);
     }
+
+    @Override
+    public String[] getDonationLinks() {
+        return new String[0];
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -209,6 +209,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return collector;
     }
 
+    @Override
+    public String[] getDonationLinks() {
+        return new String[0];
+    }
 
     @Override
     public String getErrorMessage() {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -4,22 +4,26 @@ package org.schabi.newpipe.extractor.services.youtube;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+import com.sun.org.apache.xerces.internal.xs.StringList;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.ArrayList;
 
 /*
  * Created by Christian Schabesberger on 25.07.16.
@@ -181,6 +185,29 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
     }
 
+    @Override
+    public String[] getDonationLinks() throws ParsingException {
+        try {
+            ArrayList<String> links = new ArrayList<>();
+            Element linkHolder = doc.select("div[id=\"header-links\"]").first();
+            if(linkHolder == null) {
+                // this occures if no links are embeded into the channel
+                return new String[0];
+            }
+            for(Element a : linkHolder.select("a")) {
+                String link = a.attr("abs:href");
+                if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
+                    links.add(link);
+                }
+            }
+            String[] retLinks = new String[links.size()];
+            retLinks = links.toArray(retLinks);
+            return retLinks;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get donation links", e);
+        }
+    }
+
     private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
         throws ParsingException {
         String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -528,6 +529,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return errorReason != null ? errorReason.toString() : null;
     }
 
+    @Override
+    public String[] getDonationLinks() throws ParsingException {
+        try {
+            ArrayList<String> donationLinks = new ArrayList<>();
+            for (String s : Parser.getLinksFromString(getDescription())) {
+                if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
+                    donationLinks.add(s);
+                }
+            }
+            String[] donlret = new String[donationLinks.size()];
+            donlret = donationLinks.toArray(donlret);
+            return donlret;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get donation links", e);
+        }
+    }
+
+
     /*//////////////////////////////////////////////////////////////////////////
     // Fetch page
     //////////////////////////////////////////////////////////////////////////*/

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -141,6 +141,8 @@ public abstract class StreamExtractor extends Extractor {
     public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
     public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
 
+    public abstract String[] getDonationLinks() throws ExtractionException;
+
     /**
      * Analyses the webpage's document and extracts any error message there might be.
      *

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class DonationLinkHelper {
+    public enum DonationService {
+        NO_DONATION,
+        PATREON,
+        PAYPAL
+    }
+
+
+    public static DonationService getServiceByLink(String link) throws MalformedURLException {
+        URL url = new URL(link);
+        switch (url.getHost()) {
+            case "www.patreon.com":
+                return DonationService.PATREON;
+            case "patreon.com":
+                return DonationService.PATREON;
+            case "paypal.me":
+                return DonationService.PAYPAL;
+            case "www.paypal.me":
+                return DonationService.PAYPAL;
+            default:
+                return DonationService.NO_DONATION;
+        }
+    }
+
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -1,9 +1,14 @@
 package org.schabi.newpipe.extractor.utils;
 
+import org.nibor.autolink.LinkExtractor;
+import org.nibor.autolink.LinkSpan;
+import org.nibor.autolink.LinkType;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -80,4 +85,23 @@ public class Parser {
         }
         return map;
     }
+
+    public static String[] getLinksFromString(final String txt) throws ParsingException {
+        try {
+            ArrayList<String> links = new ArrayList<>();
+            LinkExtractor linkExtractor = LinkExtractor.builder()
+                    .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
+                    .build();
+            Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
+            for(LinkSpan ls : linkss) {
+                links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
+            }
+
+            String[] linksarray = new String[links.size()];
+            linksarray = links.toArray(linksarray);
+            return linksarray;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get links from string", e);
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -103,6 +104,13 @@ public class YoutubeChannelExtractorTest {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface
+            // in order to make this work
+            assertTrue(extractor.getDonationLinks().length == 0);
+        }
     }
 
     public static class Kurzgesagt implements BaseChannelExtractorTest {
@@ -205,6 +213,11 @@ public class YoutubeChannelExtractorTest {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            assertTrue(extractor.getDonationLinks().length == 1);
+        }
     }
 
     public static class CaptainDisillusion implements BaseChannelExtractorTest {
@@ -389,6 +402,11 @@ public class YoutubeChannelExtractorTest {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            assertTrue(extractor.getDonationLinks().length == 0);
+        }
     }
 };
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java

@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+
+/*
+ * Created by Christian Schabesberger on 30.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamExtractorDonationTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+/**
+ * Test for {@link StreamExtractor}
+ */
+public class YoutubeStreamExtractorDonationTest {
+    private static YoutubeStreamExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = (YoutubeStreamExtractor) YouTube
+                .getStreamExtractor("https://www.youtube.com/watch?v=pXb3jERMoI0");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void getDonationLinksTest() throws Exception {
+        assertTrue(String.valueOf(extractor.getDonationLinks().length),
+                extractor.getDonationLinks().length == 2);
+    }
+}