be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #78 from TeamNewPipe/donation 

enable Donations for streams
This commit is contained in:
Christian Schabesberger 2018-05-07 10:08:36 +02:00 committed by GitHub
parent 1c8dafe4c6 016c2fc549
commit e5125c15f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 200 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extractor/build.gradle
View File

@ -5,6 +5,7 @@ dependencies {
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
testImplementation 'junit:junit:4.12'
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
View File

@ -35,7 +35,8 @@ public enum MediaFormat {
// audio formats
M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm"),
MP3 (0x5, "MP3", "mp3", "audio/mpeg");
MP3 (0x5, "MP3", "mp3", "audio/mpeg"),
OPUS (0x6, "opus", "opus", "audio/opus");
public final int id;
public final String name;

1
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -45,4 +45,5 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getFeedUrl() throws ParsingException;
public abstract long getSubscriberCount() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String[] getDonationLinks() throws ParsingException;
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -92,6 +92,11 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
} catch (Exception e) {
info.addError(e);
}
try {
info.setDonationLinks(extractor.getDonationLinks());
} catch (Exception e) {
info.addError(e);
}
return info;
}
@ -101,6 +106,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
private String feedUrl;
private long subscriberCount = -1;
private String description;
private String[] donationLinks;
public String getAvatarUrl() {
return avatarUrl;
@ -141,4 +147,12 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public void setDescription(String description) {
this.description = description;
}
public String[] getDonationLinks() {
return donationLinks;
}
public void setDonationLinks(String[] donationLinks) {
this.donationLinks = donationLinks;
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -133,4 +133,9 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, nextPageUrl);
}
@Override
public String[] getDonationLinks() {
return new String[0];
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -209,6 +209,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return collector;
}
@Override
public String[] getDonationLinks() {
return new String[0];
}
@Override
public String getErrorMessage() {

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -4,22 +4,26 @@ package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.sun.org.apache.xerces.internal.xs.StringList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -181,6 +185,29 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
}
@Override
public String[] getDonationLinks() throws ParsingException {
try {
ArrayList<String> links = new ArrayList<>();
Element linkHolder = doc.select("div[id=\"header-links\"]").first();
if(linkHolder == null) {
// this occures if no links are embeded into the channel
return new String[0];
}
for(Element a : linkHolder.select("a")) {
String link = a.attr("abs:href");
if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
links.add(link);
}
}
String[] retLinks = new String[links.size()];
retLinks = links.toArray(retLinks);
return retLinks;
} catch (Exception e) {
throw new ParsingException("Could not get donation links", e);
}
}
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
throws ParsingException {
String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -528,6 +529,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return errorReason != null ? errorReason.toString() : null;
}
@Override
public String[] getDonationLinks() throws ParsingException {
try {
ArrayList<String> donationLinks = new ArrayList<>();
for (String s : Parser.getLinksFromString(getDescription())) {
if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
donationLinks.add(s);
}
}
String[] donlret = new String[donationLinks.size()];
donlret = donationLinks.toArray(donlret);
return donlret;
} catch (Exception e) {
throw new ParsingException("Could not get donation links", e);
}
}
/*//////////////////////////////////////////////////////////////////////////
// Fetch page
//////////////////////////////////////////////////////////////////////////*/

2
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -141,6 +141,8 @@ public abstract class StreamExtractor extends Extractor {
public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
public abstract String[] getDonationLinks() throws ExtractionException;
/**
* Analyses the webpage's document and extracts any error message there might be.
*

31
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java Normal file
View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.utils;
import java.net.MalformedURLException;
import java.net.URL;
public class DonationLinkHelper {
public enum DonationService {
NO_DONATION,
PATREON,
PAYPAL
}
public static DonationService getServiceByLink(String link) throws MalformedURLException {
URL url = new URL(link);
switch (url.getHost()) {
case "www.patreon.com":
return DonationService.PATREON;
case "patreon.com":
return DonationService.PATREON;
case "paypal.me":
return DonationService.PAYPAL;
case "www.paypal.me":
return DonationService.PAYPAL;
default:
return DonationService.NO_DONATION;
}
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.extractor.utils;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
@ -80,4 +85,23 @@ public class Parser {
}
return map;
}
public static String[] getLinksFromString(final String txt) throws ParsingException {
try {
ArrayList<String> links = new ArrayList<>();
LinkExtractor linkExtractor = LinkExtractor.builder()
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
for(LinkSpan ls : linkss) {
links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}
String[] linksarray = new String[links.size()];
linksarray = links.toArray(linksarray);
return linksarray;
} catch (Exception e) {
throw new ParsingException("Could not get links from string", e);
}
}
}

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
@ -103,6 +104,13 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
}
@Test
public void testChannelDonation() throws Exception {
// this needs to be ignored since wed have to upgrade channel extractor to the new yt interface
// in order to make this work
assertTrue(extractor.getDonationLinks().length == 0);
}
}
public static class Kurzgesagt implements BaseChannelExtractorTest {
@ -205,6 +213,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6);
}
@Test
public void testChannelDonation() throws Exception {
assertTrue(extractor.getDonationLinks().length == 1);
}
}
public static class CaptainDisillusion implements BaseChannelExtractorTest {
@ -389,6 +402,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
}
@Test
public void testChannelDonation() throws Exception {
assertTrue(extractor.getDonationLinks().length == 0);
}
}
};

52
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java Normal file
View File

@ -0,0 +1,52 @@
package org.schabi.newpipe.extractor.services.youtube;
/*
* Created by Christian Schabesberger on 30.12.15.
*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* YoutubeStreamExtractorDonationTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link StreamExtractor}
*/
public class YoutubeStreamExtractorDonationTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=pXb3jERMoI0");
extractor.fetchPage();
}
@Test
public void getDonationLinksTest() throws Exception {
assertTrue(String.valueOf(extractor.getDonationLinks().length),
extractor.getDonationLinks().length == 2);
}
}