be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

merged upstream/dev

This commit is contained in:
yausername 2019-11-16 00:07:45 +05:30
parent f60c973928 514ed7bdc1
commit d7582734e7
58 changed files with 1232 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java
View File

@ -7,15 +7,21 @@ import java.util.Map;
import javax.annotation.Nonnull;
public class DownloadResponse {
private final int responseCode;
private final String responseBody;
private final Map<String, List<String>> responseHeaders;
public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
super();
this.responseCode = responseCode;
this.responseBody = responseBody;
this.responseHeaders = headers;
}
public int getResponseCode() {
return responseCode;
}
public String getResponseBody() {
return responseBody;
}
@ -33,5 +39,4 @@ public class DownloadResponse {
else
return cookies;
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -60,6 +60,11 @@ public interface Downloader {
*/
String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, Localization localization)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;

9
extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java
View File

@ -21,7 +21,14 @@ package org.schabi.newpipe.extractor.exceptions;
*/
public class ReCaptchaException extends ExtractionException {
public ReCaptchaException(String message) {
private String url;
public ReCaptchaException(String message, String url) {
super(message);
this.url = url;
}
public String getUrl() {
return url;
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java
View File

@ -24,7 +24,9 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
return itemObject.getString("avatar_url", "");
String avatarUrl = itemObject.getString("avatar_url", "");
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
}
@Override

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -7,9 +7,12 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -21,40 +24,59 @@ import java.io.IOException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19
private static String clientId;
private SoundcloudParsingHelper() {
}
public static String clientId() throws ReCaptchaException, IOException, RegexException {
public static String clientId() throws ExtractionException, IOException {
if (clientId != null && !clientId.isEmpty()) return clientId;
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://soundcloud.com");
Document doc = Jsoup.parse(response);
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
final String clientIdPattern = ",client_id:\"(.*?)\"";
try {
final HashMap<String, String> headers = new HashMap<>();
headers.put("Range", "bytes=0-16384");
String js = dl.download(jsElement.attr("src"), headers);
return clientId = Parser.matchGroup1(clientIdPattern, js);
} catch (IOException | RegexException ignored) {
// Ignore it and proceed to download the whole js file
clientId = HARDCODED_CLIENT_ID;
if (checkIfHardcodedClientIdIsValid(dl)) {
return clientId;
}
String js = dl.download(jsElement.attr("src"));
return clientId = Parser.matchGroup1(clientIdPattern, js);
final DownloadResponse download = dl.get("https://soundcloud.com");
String response = download.getResponseBody();
final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(response);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final HashMap<String, String> headers = new HashMap<>();
headers.put("Range", "bytes=0-16384");
for (Element element : possibleScripts) {
final String srcUrl = element.attr("src");
if (srcUrl != null && !srcUrl.isEmpty()) {
try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
} catch (RegexException ignored) {
// Ignore it and proceed to try searching other script
}
}
}
// Officially give up
throw new ExtractionException("Couldn't extract client id");
}
static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
// Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
return dl.head(apiUrl).getResponseCode() == 200;
}
public static String toDateString(String time) throws ParsingException {
@ -79,7 +101,7 @@ public class SoundcloudParsingHelper {
*
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
@ -113,7 +135,10 @@ public class SoundcloudParsingHelper {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"));
return Parser.matchGroup1(",\"id\":(.*?),", response);
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
}
/**

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -71,13 +71,15 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
final String thumbnailUrl = item.getThumbnailUrl();
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
return thumbnailUrl;
String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
return thumbnailUrlBetterResolution;
}
} catch (Exception ignored) {
}
}
return artworkUrl;
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
@Override

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -32,7 +32,10 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
// Over-engineering at its finest
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) return artworkUrl;
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
}
try {
@ -42,8 +45,11 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
final String url = trackObject.getString(ARTWORK_URL_KEY, "");
if (!url.isEmpty()) return url;
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
}
// Then look for track creator avatar url

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
@ -48,10 +49,10 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
} catch (IOException e) {
throw new ParsingException("Could not get client id", e);
} catch (ReCaptchaException e) {
throw new ParsingException("ReCaptcha required", e);
} catch (IOException | ExtractionException e) {
throw new ParsingException("Could not get client id", e);
}
}

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -95,6 +95,7 @@ public class SoundcloudService extends StreamingService {
try {
list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot");
list.setDefaultKiosk("New & hot");
} catch (Exception e) {
throw new ExtractionException(e);
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -57,7 +57,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() {
return track.getString("artwork_url", "");
String artworkUrl = track.getString("artwork_url", "");
if (artworkUrl.isEmpty()) {
artworkUrl = track.getObject("user").getString("avatar_url", "");
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
@Nonnull

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -52,7 +52,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public String getThumbnailUrl() {
return itemObject.getString("artwork_url");
String artworkUrl = itemObject.getString("artwork_url", "");
if (artworkUrl.isEmpty()) {
artworkUrl = itemObject.getObject("user").getString("avatar_url");
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
@Override

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
@ -47,8 +49,9 @@ import java.util.ArrayList;
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
private Document doc;
@ -59,8 +62,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);
final DownloadResponse response = downloader.get(channelUrl);
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
}
@Override
@ -72,7 +75,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getUrl() throws ParsingException {
try {
return "https://www.youtube.com/channel/" + getId();
return CHANNEL_URL_BASE + getId();
} catch (ParsingException e) {
return super.getUrl();
}
@ -81,6 +84,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
try {
return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
} catch (Exception ignored) {}
// fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
try {
Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
@ -134,10 +142,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (el != null) {
String elTitle = el.attr("title");
try {
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
return Utils.mixedNumberWordToLong(elTitle);
} catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -5,6 +5,9 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
* Created by Christian Schabesberger on 12.02.17.
*
@ -53,8 +56,26 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getUrl() throws ParsingException {
try {
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
.attr("abs:data-href");
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
if (match.matches()) {
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
}
} catch(Exception ignored) {}
// fallback method for channels without "Subscribe" button (or just in case yt changes things)
// provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
try {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get channel url", e);
}
}
@Override

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -38,7 +38,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
return "";
}
}
@ -95,7 +95,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
return "";
}
}
@ -104,7 +104,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) {
throw new ParsingException("Could not get author endpoint", e);
return "";
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -35,8 +36,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
final String url = getUrl();
final DownloadResponse response = downloader.get(url);
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@Override
@ -50,7 +52,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get playlist name");
throw new ParsingException("Could not get playlist name", e);
}
}
@ -59,7 +61,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
} catch (Exception e) {
throw new ParsingException("Could not get playlist thumbnail");
throw new ParsingException("Could not get playlist thumbnail", e);
}
}
@ -72,9 +74,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
try {
return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
return YoutubeChannelExtractor.CHANNEL_URL_BASE +
doc.select("button[class*=\"yt-uix-subscription-button\"]")
.first().attr("data-channel-external-id");
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
throw new ParsingException("Could not get playlist uploader url", e);
}
}
@ -83,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
throw new ParsingException("Could not get playlist uploader name", e);
}
}
@ -92,7 +96,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar");
throw new ParsingException("Could not get playlist uploader avatar", e);
}
}
@ -248,6 +252,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
// this url is not always in the form "/channel/..."
// sometimes Youtube provides urls in the from "/user/..."
return getUploaderLink().attr("abs:href");
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -49,10 +49,11 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getUrl() throws ParsingException {
try {
final Element div = el.select("div[class=\"yt-lockup-meta\"]").first();
final Element a = el.select("div[class=\"yt-lockup-meta\"]")
.select("ul[class=\"yt-lockup-meta-info\"]")
.select("li").select("a").first();
if(div != null) {
final Element a = div.select("a").first();
if(a != null) {
return a.attr("abs:href");
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
@ -52,13 +54,9 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String site;
final String url = getUrl();
//String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
site = downloader.download(url, getLocalization());
doc = Jsoup.parse(site, url);
final DownloadResponse response = downloader.get(url, getLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@Override

344
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
@ -30,11 +31,13 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
* YoutubeStreamExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
@ -64,12 +67,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
public class SubtitlesException extends ContentNotAvailableException {
SubtitlesException(String message, Throwable cause) {
super(message, cause);
@ -83,6 +80,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private JsonObject playerArgs;
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
@Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
@ -162,14 +160,54 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
// onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;"
// :00 is NOT recognized as a timestamp in description or comments.
// 0:00 is recognized in both description and comments.
// https://www.youtube.com/watch?v=4cccfDXu1vA
private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile(
"seekTo\\("
+ "(?:(\\d+)\\*3600\\+)?" // hours?
+ "(\\d+)\\*60\\+" // minutes
+ "(\\d+)" // seconds
+ "\\)");
@SafeVarargs
private static <T> T coalesce(T... args) {
for (T arg : args) {
if (arg != null) return arg;
}
throw new IllegalArgumentException("all arguments to coalesce() were null");
}
private String parseHtmlAndGetFullLinks(String descriptionHtml)
throws MalformedURLException, UnsupportedEncodingException, ParsingException {
final Document description = Jsoup.parse(descriptionHtml, getUrl());
for(Element a : description.select("a")) {
final URL redirectLink = new URL(
a.attr("abs:href"));
final String queryString = redirectLink.getQuery();
if(queryString != null) {
final String rawUrl = a.attr("abs:href");
final URL redirectLink = new URL(rawUrl);
final Matcher onClickTimestamp;
final String queryString;
if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick")))
.find()) {
a.removeAttr("onclick");
String hours = coalesce(onClickTimestamp.group(1), "0");
String minutes = onClickTimestamp.group(2);
String seconds = onClickTimestamp.group(3);
int timestamp = 0;
timestamp += Integer.parseInt(hours) * 3600;
timestamp += Integer.parseInt(minutes) * 60;
timestamp += Integer.parseInt(seconds);
String setTimestamp = "&t=" + timestamp;
// Even after clicking https://youtu.be/...?t=6,
// getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=.
a.attr("href", getUrl() + setTimestamp);
} else if((queryString = redirectLink.getQuery()) != null) {
// if the query string is null we are not dealing with a redirect link,
// so we don't need to override it.
final String link =
@ -179,11 +217,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// if link is null the a tag is a hashtag.
// They refer to the youtube search. We do not handle them.
a.text(link);
a.attr("href", link);
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
a.text(redirectLink.toString());
a.attr("href", redirectLink.toString());
}
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
a.text(redirectLink.toString());
a.attr("href", redirectLink.toString());
}
}
return description.select("body").first().html();
@ -206,29 +248,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getLength() throws ParsingException {
assertPageFetched();
if(playerArgs != null) {
try {
long returnValue = Long.parseLong(playerArgs.get("length_seconds") + "");
if (returnValue >= 0) return returnValue;
} catch (Exception ignored) {
// Try other method...
}
}
String lengthString = videoInfoPage.get("length_seconds");
// try getting duration from playerargs
try {
return Long.parseLong(lengthString);
} catch (Exception ignored) {
// Try other method...
}
// TODO: 25.11.17 Implement a way to get the length for age restricted videos #44
try {
// Fallback to HTML method
return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first()
.attr("aria-valuemax"));
String durationMs = playerResponse
.getObject("streamingData")
.getArray("formats")
.getObject(0)
.getString("approxDurationMs");
return Long.parseLong(durationMs)/1000;
} catch (Exception e) {
throw new ParsingException("Could not get video length", e);
}
//try getting value from age gated video
try {
String duration = playerResponse
.getObject("videoDetails")
.getString("lengthSeconds");
return Long.parseLong(duration);
} catch (Exception e) {
throw new ParsingException("Every methode to get the duration has failed: ", e);
}
}
@ -385,31 +424,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getHlsUrl() throws ParsingException {
assertPageFetched();
try {
String hlsvp = "";
if (playerArgs != null) {
if( playerArgs.isString("hlsvp") ) {
hlsvp = playerArgs.getString("hlsvp", "");
}else {
hlsvp = JsonParser.object()
.from(playerArgs.getString("player_response", "{}"))
.getObject("streamingData", new JsonObject())
.getString("hlsManifestUrl", "");
}
}
return hlsvp;
try {
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
} catch (Exception e) {
if (playerArgs != null && playerArgs.isString("hlsvp")) {
return playerArgs.getString("hlsvp");
} else {
throw new ParsingException("Could not get hls manifest url", e);
}
}
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
public List<AudioStream> getAudioStreams() throws ExtractionException {
assertPageFetched();
List<AudioStream> audioStreams = new ArrayList<>();
try {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
ItagItem itag = entry.getValue();
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
@ -425,11 +457,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
public List<VideoStream> getVideoStreams() throws ExtractionException {
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
try {
for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
ItagItem itag = entry.getValue();
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
@ -449,7 +481,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
List<VideoStream> videoOnlyStreams = new ArrayList<>();
try {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
ItagItem itag = entry.getValue();
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
@ -486,7 +518,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
try {
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
return StreamType.LIVE_STREAM;
}
} catch (Exception e) {
@ -538,22 +570,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
*/
@Override
public String getErrorMessage() {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
StringBuilder errorReason;
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
if (errorElement == null) {
errorReason = null;
} else {
String errorMessage = errorElement.text();
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else if (errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
}
return errorReason != null ? errorReason.toString() : null;
}
@ -562,8 +593,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Fetch page
//////////////////////////////////////////////////////////////////////////*/
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
private static final String ADAPTIVE_FMTS = "adaptive_fmts";
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
@ -571,7 +602,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX =
"(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
private final static String DECRYPTION_AKAMAIZED_STRING_REGEX =
"yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX =
@ -581,22 +612,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private String pageHtml = null;
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
if (pageHtml == null) {
pageHtml = downloader.download(verifiedUrl);
}
return pageHtml;
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String pageContent = getPageHtml(downloader);
doc = Jsoup.parse(pageContent, getUrl());
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
final DownloadResponse response = downloader.get(verifiedUrl);
pageHtml = response.getResponseBody();
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
final String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.download(videoInfoUrl);
@ -604,11 +629,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = info.url;
isAgeRestricted = true;
} else {
final JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
final JsonObject ytPlayerConfig = getPlayerConfig();
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
playerResponse = getPlayerResponse();
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
@ -619,15 +645,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
private JsonObject getPlayerConfig() throws ParsingException {
try {
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
return JsonParser.object().from(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch (errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
@ -670,6 +694,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private JsonObject getPlayerResponse() throws ParsingException {
try {
String playerResponseStr;
if(playerArgs != null) {
playerResponseStr = playerArgs.getString("player_response");
} else {
playerResponseStr = videoInfoPage.get("player_response");
}
return JsonParser.object().from(playerResponseStr);
} catch (Exception e) {
throw new ParsingException("Could not parse yt player response", e);
}
}
@Nonnull
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
try {
@ -685,16 +723,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = HTTPS + playerUrl;
}
try {
// Get embed sts
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
return new EmbeddedInfo(playerUrl, sts);
} catch (Exception i) {
// if it failes we simply reply with no sts as then it does not seem to be necessary
return new EmbeddedInfo(playerUrl, "");
}
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}
@ -708,15 +749,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
final String playerCode = downloader.download(playerUrl);
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
final String decryptionFunctionName;
if (Parser.isMatch(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode)) {
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
} else if (Parser.isMatch(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode)) {
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
} else {
decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
}
final String functionPattern = "("
+ decryptionFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
@ -757,30 +791,38 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString();
}
private String getDecryptionFuncName(String playerCode) throws DecryptException {
String decryptionFunctionName;
// Cascading things in catch is ugly, but its faster than running a match before getting the actual name
// to se if the function can actually be found with the given regex.
// However if this cascading should propably be cleaned up somehow as it looks a bit weird.
try {
decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
} catch (Parser.RegexException re) {
try {
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
} catch (Parser.RegexException re2) {
try {
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
} catch (Parser.RegexException re3) {
throw new DecryptException("Could not find decrypt function with any of the given patterns.", re);
}
}
}
return decryptionFunctionName;
}
@Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
// If the video is age restricted getPlayerConfig will fail
if(isAgeRestricted) return Collections.emptyList();
final JsonObject playerConfig;
try {
playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
} catch (IOException | ExtractionException e) {
throw new SubtitlesException("Unable to download player configs", e);
}
final String playerResponse = playerConfig.getObject("args", new JsonObject())
.getString("player_response");
final JsonObject captions;
try {
if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
if (!playerResponse.has("captions")) {
// Captions does not exist
return Collections.emptyList();
}
captions = JsonParser.object().from(playerResponse).getObject("captions");
} catch (JsonParserException e) {
throw new SubtitlesException("Unable to parse subtitles listing", e);
}
captions = playerResponse.getObject("captions");
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
@ -849,39 +891,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
String encodedUrlMap = "";
if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
} else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
JsonObject streamingData = playerResponse.getObject("streamingData");
if (!streamingData.has(streamingDataKey)) {
return urlAndItags;
}
for (String url_data_str : encodedUrlMap.split(",")) {
try {
// This loop iterates through multiple streams, therefore tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
int itag = Integer.parseInt(tags.get("itag"));
JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
if (ItagItem.isSupported(itag)) {
try {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// this url has an encrypted signature
Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
}
urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {
}
} catch (DecryptException e) {
throw e;
} catch (Exception ignored) {
}
}
@ -954,4 +993,61 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
};
}
@Nonnull
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
int p = script.indexOf("ytplayer.config");
if (p == -1) {
return Collections.emptyList();
}
p = script.indexOf('{', p);
int e = script.indexOf("ytplayer.load", p);
if (e == -1) {
return Collections.emptyList();
}
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
final String url = spec[0];
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8) {
continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
final int frameHeight = Integer.parseInt(parts[1]);
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls;
if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j)));
}
} else {
urls = Collections.singletonList(baseUrl);
}
result.add(new Frameset(
urls,
frameWidth,
frameHeight,
totalCount,
framesPerPageX,
framesPerPageY
));
}
result.trimToSize();
return result;
} catch (Exception e) {
throw new ExtractionException(e);
}
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -61,7 +61,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUrl() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
@ -72,7 +72,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getName() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
@ -107,6 +107,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
// this url is not always in the form "/channel/..."
// sometimes Youtube provides urls in the from "/user/..."
try {
try {
return item.select("div[class=\"yt-lockup-byline\"]").first()
@ -119,7 +121,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
.text().split(" - ")[0];
} catch (Exception e) {
System.out.println(item.html());
throw new ParsingException("Could not get uploader", e);
throw new ParsingException("Could not get uploader url", e);
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View File

@ -63,16 +63,10 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
String title = outline.attr("title");
String xmlUrl = outline.attr("abs:xmlUrl");
if (title.isEmpty() || xmlUrl.isEmpty()) {
throw new InvalidSourceException("document has invalid entries");
}
try {
String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
} catch (Parser.RegexException e) {
throw new InvalidSourceException("document has invalid entries", e);
}
} catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
}
return result;

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -24,12 +24,14 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
@ -56,8 +58,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
url += "?gl=" + contentCountry;
}
String pageContent = downloader.download(url);
doc = Jsoup.parse(pageContent, url);
final DownloadResponse response = downloader.get(url);
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@Override
@ -126,6 +128,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
}
private Element getUploaderLink() {
// this url is not always in the form "/channel/..."
// sometimes Youtube provides urls in the from "/user/..."
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
return uploaderEl.select("a").first();
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View File

@ -46,7 +46,8 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();
if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) {
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) {
throw new ParsingException("the URL given is not a Youtube-URL");
}

54
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -1,7 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.net.URL;
@ -30,40 +34,42 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
private static boolean isHTTP(URL url) {
// make sure its http or https
String protocol = url.getProtocol();
if (!protocol.equals("http") && !protocol.equals("https")) {
return false;
private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
"form[action*=\"/das_captcha\"]",
"input[name*=\"action_recaptcha_verify\"]"
};
public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.getResponseBody(), url);
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
if (!document.select(detectionSelector).isEmpty()) {
throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url);
}
}
boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
boolean setsNoPort = url.getPort() == -1;
return setsNoPort || usesDefaultPort;
return document;
}
public static boolean isYoutubeURL(URL url) {
// make sure its http or https
if (!isHTTP(url))
return false;
// make sure its a known youtube url
String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com");
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
}
public static boolean isYoutubeALikeURL(URL url) {
// make sure its http or https
if (!isHTTP(url))
return false;
// make sure its a known youtube url
public static boolean isYoutubeServiceURL(URL url) {
String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("www.youtube-nocookie.com")
|| host.equalsIgnoreCase("youtu.be") || host.equalsIgnoreCase("hooktube.com");
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
}
public static boolean isHooktubeURL(URL url) {
String host = url.getHost();
return host.equalsIgnoreCase("hooktube.com");
}
public static boolean isInvidioURL(URL url) {
String host = url.getHost();
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
}
public static long parseDurationString(String input)

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -25,10 +25,16 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
try {
URL urlObj = Utils.stringToURL(url);
if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) {
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj))) {
throw new ParsingException("the url given is not a Youtube-URL");
}
String path = urlObj.getPath();
if (!path.equals("/watch" ) && !path.equals("/playlist")) {
throw new ParsingException("the url given is neither a video nor a playlist URL");
}
String listID = Utils.getQueryValue(urlObj, "list");
if (listID == null) {

49
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View File

@ -60,7 +60,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
URI uri = new URI(urlString);
String scheme = uri.getScheme();
if (scheme != null && scheme.equals("vnd.youtube")) {
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
String schemeSpecificPart = uri.getSchemeSpecificPart();
if (schemeSpecificPart.startsWith("//")) {
urlString = "https:" + schemeSpecificPart;
@ -85,7 +85,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
path = path.substring(1);
}
if (!YoutubeParsingHelper.isYoutubeALikeURL(url)) {
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
YoutubeParsingHelper.isInvidioURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString);
}
@ -112,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "YOUTUBE.COM":
case "WWW.YOUTUBE.COM":
case "M.YOUTUBE.COM": {
case "M.YOUTUBE.COM":
case "MUSIC.YOUTUBE.COM": {
if (path.equals("attribution_link")) {
String uQueryValue = Utils.getQueryValue(url, "u");
@ -147,6 +150,34 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
}
case "HOOKTUBE.COM": {
if (path.startsWith("v/")) {
String id = path.substring("v/".length());
return assertIsID(id);
}
if (path.startsWith("watch/")) {
String id = path.substring("watch/".length());
return assertIsID(id);
}
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
}
case "WWW.INVIDIO.US":
case "DEV.INVIDIO.US":
case "INVIDIO.US":
case "INVIDIOUS.SNOPYTA.ORG":
case "DE.INVIDIOUS.SNOPYTA.ORG":
case "FI.INVIDIOUS.SNOPYTA.ORG":
case "VID.WXZM.SX":
case "INVIDIOUS.KABI.TK":
case "INVIDIOU.SH":
case "WWW.INVIDIOU.SH":
case "NO.INVIDIOU.SH":
case "INVIDIOUS.ENKIRTON.NET":
case "TUBE.POAL.CO":
case "INVIDIOUS.13AD.DE":
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
if (path.equals("watch")) {
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
@ -158,20 +189,10 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsID(id);
}
if (path.startsWith("v/")) {
String id = path.substring("v/".length());
return assertIsID(id);
}
if (path.startsWith("watch/")) {
String id = path.substring("watch/".length());
return assertIsID(id);
}
}
break;
}
}
throw new ParsingException("Error no suitable url: " + urlString);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
View File

@ -48,6 +48,6 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
}
String urlPath = urlObj.getPath();
return YoutubeParsingHelper.isYoutubeURL(urlObj) && urlPath.equals("/feed/trending");
return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending");
}
}

66
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java Normal file
View File

@ -0,0 +1,66 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.util.Collection;
import java.util.List;
public final class Frameset {
private List<String> urls;
private int frameWidth;
private int frameHeight;
private int totalCount;
private int framesPerPageX;
private int framesPerPageY;
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
this.urls = urls;
this.totalCount = totalCount;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
this.framesPerPageY = framesPerPageY;
}
/**
* @return list of urls to images with frames
*/
public List<String> getUrls() {
return urls;
}
/**
* @return total count of frames
*/
public int getTotalCount() {
return totalCount;
}
/**
* @return maximum frames count by x
*/
public int getFramesPerPageX() {
return framesPerPageX;
}
/**
* @return maximum frames count by y
*/
public int getFramesPerPageY() {
return framesPerPageY;
}
/**
* @return width of a one frame, in pixels
*/
public int getFrameWidth() {
return frameWidth;
}
/**
* @return height of a one frame, in pixels
*/
public int getFrameHeight() {
return frameHeight;
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -30,7 +30,10 @@ import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
@ -256,7 +259,18 @@ public abstract class StreamExtractor extends Extractor {
public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
/**
* Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
* Should return a list of Frameset object that contains preview of stream frames
* @return list of preview frames or empty list if frames preview is not supported or not found for specified stream
* @throws IOException
* @throws ExtractionException
*/
@Nonnull
public List<Frameset> getFrames() throws IOException, ExtractionException {
return Collections.emptyList();
}
/**
* Should analyse the webpage's document and extracts any error message there might be.
*
* @return Error message; null if there is no error message.
*/

2
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -123,8 +123,6 @@ public class DashMpdParser {
dashDoc = downloader.download(streamInfo.getDashMpdUrl());
} catch (IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
try {

45
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -27,6 +27,38 @@ public class Utils {
return toRemove.replaceAll("\\D+", "");
}
/**
* <p>Convert a mixed number word to a long.</p>
* <p>Examples:</p>
* <ul>
* <li>123 -&gt; 123</li>
* <li>1.23K -&gt; 1230</li>
* <li>1.23M -&gt; 1230000</li>
* </ul>
* @param numberWord string to be converted to a long
* @return a long
* @throws NumberFormatException
* @throws ParsingException
*/
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
String multiplier = "";
try {
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
} catch(ParsingException ignored) {}
double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)
.replace(",", "."));
switch (multiplier.toUpperCase()) {
case "K":
return (long) (count * 1e3);
case "M":
return (long) (count * 1e6);
case "B":
return (long) (count * 1e9);
default:
return (long) (count);
}
}
/**
* Check if the url matches the pattern.
*
@ -121,6 +153,19 @@ public class Utils {
}
}
public static boolean isHTTP(URL url) {
// make sure its http or https
String protocol = url.getProtocol();
if (!protocol.equals("http") && !protocol.equals("https")) {
return false;
}
boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
boolean setsNoPort = url.getPort() == -1;
return setsNoPort || usesDefaultPort;
}
public static String removeUTF8BOM(String s) {
if (s.startsWith("\uFEFF")) {
s = s.substring(1);

38
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Collections.singletonList;
/*
* Created by Christian Schabesberger on 28.01.16.
*
@ -129,7 +131,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
@ -172,6 +174,36 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return dl(con);
}
@Override
public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException {
final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection();
try {
con.setRequestMethod("HEAD");
setDefaults(con);
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
}
return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields());
}
@Override
public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept-Language", singletonList(localization.getLanguage()));
return get(siteUrl, new DownloadRequest(null, requestHeaders));
}
@Override
public DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
@ -183,7 +215,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
}
String responseBody = dl(con);
return new DownloadResponse(responseBody, con.getHeaderFields());
return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields());
}
@Override
@ -219,6 +251,6 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
sb.append(inputLine);
}
}
return new DownloadResponse(sb.toString(), con.getHeaderFields());
return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java
View File

@ -36,7 +36,7 @@ public class MediaCCCOggTest {
@Test
public void getAudioStreamsContainOgg() throws Exception {
for(AudioStream stream : extractor.getAudioStreams()) {
System.out.println(stream.getFormat());
assertEquals("OGG", stream.getFormat().toString());
}
}
}

26
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -25,7 +25,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
* Test for {@link PeertubeChannelExtractor}
*/
public class PeertubeChannelExtractorTest {
public static class LilUzi implements BaseChannelExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -34,7 +34,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org");
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
extractor.fetchPage();
}
@ -49,22 +49,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("Noureddine HADDAG", extractor.getName());
assertEquals("The KDE Community", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("root@tube.openalgeria.org", extractor.getId());
assertEquals("kde", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -112,7 +112,7 @@ public class PeertubeChannelExtractorTest {
}
}
public static class DubMatix implements BaseChannelExtractorTest {
public static class Booteille implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -121,7 +121,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch");
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
extractor.fetchPage();
}
@ -146,22 +146,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("France Inter", extractor.getName());
assertEquals("booteille", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("franceinter@tube.kdy.ch", extractor.getId());
assertEquals("booteille", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -205,7 +205,7 @@ public class PeertubeChannelExtractorTest {
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
}
}
}

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View File

@ -32,7 +32,7 @@ public class PeertubeStreamExtractorDefaultTest {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
extractor.fetchPage();
}
@ -44,39 +44,39 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "The Internet's Own Boy");
assertEquals(extractor.getName(), "Power Corrupts the Best");
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "The story of programming prodigy and information activist Aaron Swartz, who took his own life at the age of 26.");
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "root");
assertEquals(extractor.getUploaderName(), "Rowsedower");
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 6299);
assertEquals(extractor.getLength(), 269);
}
@Test
public void testGetViewCount() throws ParsingException {
assertTrue(Long.toString(extractor.getViewCount()),
extractor.getViewCount() > 700);
extractor.getViewCount() > 10);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertEquals("2017-10-17", extractor.getUploadDate());
assertEquals("2018-09-30", extractor.getUploadDate());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@peertube2.cpy.re", extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
}
@Test

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java
View File

@ -25,23 +25,21 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy");
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde");
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet%27s+own+boy&start=12&count=12", extractor.getNextPageUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl());
}
@Test
public void testResultList_FirstElement() {
InfoItem firstInfoItem = itemsPage.getItems().get(0);
// THe channel should be the first item
assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName());
assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl());
assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde"));
}
@Test
@ -83,11 +81,11 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
@Test
public void testId() throws Exception {
assertEquals("internet's own boy", extractor.getId());
assertEquals("kde", extractor.getId());
}
@Test
public void testName() {
assertEquals("internet's own boy", extractor.getName());
assertEquals("kde", extractor.getName());
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -36,7 +36,6 @@ public class SoundcloudChartsExtractorTest {
assertNotNull(NewPipe.getDownloader());
}
@Ignore
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getName(), "Top 50");

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java
View File

@ -1,18 +1,24 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.*;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
public class SoundcloudParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
}
@Test
public void assertThatHardcodedClientIdIsValid() throws Exception {
assertTrue("Hardcoded client id is not valid anymore",
SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(Downloader.getInstance()));
}
@Test
public void resolveUrlWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.hamcrest.CoreMatchers;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractorTest {
}
}
public static class RandomHouseDanceMusic implements BasePlaylistExtractorTest {
public static class RandomHouseMusic implements BasePlaylistExtractorTest {
private static SoundcloudPlaylistExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2");
.getPlaylistExtractor("https://soundcloud.com/micky96/sets/house");
extractor.fetchPage();
}
@ -141,22 +142,22 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testName() {
assertEquals("House, Electro , Dance Music 2", extractor.getName());
assertEquals("House", extractor.getName());
}
@Test
public void testId() {
assertEquals("310980722", extractor.getId());
assertEquals("123062856", extractor.getId());
}
@Test
public void testUrl() throws Exception {
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl());
assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws Exception {
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl());
assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -182,7 +183,7 @@ public class SoundcloudPlaylistExtractorTest {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Ignore
@Ignore("not implemented")
@Test
public void testBannerUrl() {
assertIsSecureUrl(extractor.getBannerUrl());
@ -192,12 +193,12 @@ public class SoundcloudPlaylistExtractorTest {
public void testUploaderUrl() {
final String uploaderUrl = extractor.getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader"));
assertThat(uploaderUrl, CoreMatchers.containsString("micky96"));
}
@Test
public void testUploaderName() {
assertEquals("Gosu", extractor.getUploaderName());
assertEquals("_mickyyy", extractor.getUploaderName());
}
@Test
@ -226,6 +227,7 @@ public class SoundcloudPlaylistExtractorTest {
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
@ -265,11 +267,14 @@ public class SoundcloudPlaylistExtractorTest {
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
}
//TODO: FUCK THIS: This triggers a 500 at sever
@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());
@ -284,6 +289,7 @@ public class SoundcloudPlaylistExtractorTest {
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testThumbnailUrl() {
assertIsSecureUrl(extractor.getThumbnailUrl());

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -39,27 +39,27 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69");
assertEquals(extractor.getTimeStamp() + "", "69");
assertEquals("69", extractor.getTimeStamp() + "");
}
@Test
public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Do What I Want [Produced By Maaly Raw + Don Cannon]");
assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName());
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "The Perfect LUV Tape®️");
assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "LIL UZI VERT");
assertEquals("LIL UZI VERT", extractor.getUploaderName());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 175);
assertEquals(175, extractor.getLength());
}
@Test

101
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -109,6 +109,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6);
}
}
@ -199,6 +200,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6);
}
}
@ -394,6 +396,100 @@ public class YoutubeChannelExtractorTest {
}
}
// this channel has no "Subscribe" button
public static class EminemVEVO implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertEquals("EminemVEVO", extractor.getName());
}
@Test
public void testId() throws Exception {
assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, YouTube.getServiceId());
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws Exception {
final String description = extractor.getDescription();
assertTrue(description, description.contains("Eminem on Vevo"));
}
@Test
public void testAvatarUrl() throws Exception {
String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
}
@Test
public void testBannerUrl() throws Exception {
String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt3"));
}
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws Exception {
// there is no "Subscribe" button
long subscribers = extractor.getSubscriberCount();
assertEquals("Wrong subscriber count", -1, subscribers);
}
}
public static class RandomChannel implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@ -485,8 +581,9 @@ public class YoutubeChannelExtractorTest {
@Test
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
long subscribers = extractor.getSubscriberCount();
assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
}
}
};
}

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java
View File

@ -37,6 +37,12 @@ public class YoutubeChannelLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA"));
assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
}
@Test
@ -53,5 +59,11 @@ public class YoutubeChannelLinkHandlerFactoryTest {
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId());
assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
}
}

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -27,18 +27,18 @@ public class YoutubeCommentsExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
.getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
boolean result;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "Tsuki blyat");
result = findInComments(comments, "s1ck m3m3");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "Tsuki blyat");
result = findInComments(comments, "s1ck m3m3");
}
assertTrue(result);
@ -47,14 +47,14 @@ public class YoutubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
assertTrue("the dark side of YouTube...".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Tsuki blyat");
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o");
assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
String nextPage = commentsInfo.getNextPageUrl();
while (!StringUtil.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Tsuki blyat");
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl();
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -100,7 +100,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl());
}
@Test
@ -185,8 +185,8 @@ public class YoutubePlaylistExtractorTest {
public void testMoreRelatedItems() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage
= defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
// Test for 2 more levels
// test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors());
@ -214,7 +214,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl());
}
@Test

107
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,107 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
/**
* Test for {@link YoutubePlaylistLinkHandlerFactory}
*/
public class YoutubePlaylistLinkHandlerFactoryTest {
private static YoutubePlaylistLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = YoutubePlaylistLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
}
@Test(expected = IllegalArgumentException.class)
public void getIdWithNullAsUrl() throws ParsingException {
linkHandler.fromId(null);
}
@Test
public void getIdfromYt() throws Exception {
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId());
}
@Test
public void testAcceptYtUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
}
@Test
public void testDeniesInvalidYtUrl() throws ParsingException {
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("www.youtube.com:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("youtube . com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
}
@Test
public void testAcceptInvidioUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
assertTrue(linkHandler.acceptUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
}
@Test
public void testDeniesInvalidInvidioUrl() throws ParsingException {
assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("invidio.us:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("invidio . us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
}
@Test
public void testGetInvidioIdfromUrl() throws ParsingException {
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
}
}

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java
View File

@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
}
@Test
@ -98,13 +99,14 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
}
@Test
public void testAcceptHookUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
assertTrue(linkHandler.acceptUrl("hooktube.com/watch/3msbfr6pBNE"));
@ -115,10 +117,31 @@ public class YoutubeStreamLinkHandlerFactoryTest {
@Test
public void testGetHookIdfromUrl() throws ParsingException {
assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId());
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId());
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId());
}
@Test
public void testAcceptInvidioUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE"));
}
@Test
public void testGetInvidioIdfromUrl() throws ParsingException {
assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId());
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId());
}
}

33
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -59,15 +59,38 @@ public class YoutubeSubscriptionExtractorTest {
assertTrue(items.isEmpty());
}
@Test
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
"</outline></body></opml>";
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
}
@Test
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
"<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
"</outline></body></opml>";
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue(items.isEmpty());
}
@Test
public void testInvalidSourceException() {
List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"<opml><body></body></opml>",
"<opml><body><outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/></outline></body></opml>",
"<opml><body><outline><outline text=\"invalid\" title=\"url\" type=\"rss\"" +
" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/></outline></body></opml>",
"",
null,
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
@ -78,11 +101,11 @@ public class YoutubeSubscriptionExtractorTest {
if (invalidContent != null) {
byte[] bytes = invalidContent.getBytes("UTF-8");
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} else {
subscriptionExtractor.fromInputStream(null);
fail("Extracting from null String didn't throw an exception");
}
fail("didn't throw exception");
} catch (Exception e) {
// System.out.println(" -> " + e);
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java
View File

@ -69,6 +69,10 @@ public class YoutubeTrendingLinkHandlerFactoryTest {
assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending"));
assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending"));
assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending"));
assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending"));
assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending"));
assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending"));
assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia"));
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending"));

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube.search;
import org.hamcrest.CoreMatchers;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
@ -53,6 +55,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl());
}
@Ignore
@Test
public void testOnlyContainChannels() {
for(InfoItem item : itemsPage.getItems()) {
@ -61,4 +64,19 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
}
}
}
@Test
public void testChannelUrl() {
for(InfoItem item : itemsPage.getItems()) {
if (item instanceof ChannelInfoItem) {
ChannelInfoItem channel = (ChannelInfoItem) item;
if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie
assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl());
} else {
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
}
}
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java
View File

@ -73,7 +73,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
assertTrue((firstInfoItem instanceof ChannelInfoItem)
|| (secondInfoItem instanceof ChannelInfoItem));
assertEquals("name", "PewDiePie", channelItem.getName());
assertEquals("url","https://www.youtube.com/user/PewDiePie", channelItem.getUrl());
assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl());
}
@Test

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java → extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
@ -71,10 +71,9 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
assertFalse(extractor.getUploaderName().isEmpty());
}
@Ignore // Currently there is no way get the length from restricted videos
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
assertEquals(1789, extractor.getLength());
}
@Test
@ -97,8 +96,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
// FIXME: 25.11.17 Are there no streams or are they not listed?
@Ignore
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audio streams are not always necessary

13
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java → extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
@ -71,10 +71,9 @@ public class YoutubeStreamExtractorControversialTest {
assertFalse(extractor.getUploaderName().isEmpty());
}
@Ignore // Currently there is no way get the length from restricted videos
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
assertEquals(219, extractor.getLength());
}
@Test
@ -97,8 +96,6 @@ public class YoutubeStreamExtractorControversialTest {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
// FIXME: 25.11.17 Are there no streams or are they not listed?
@Ignore
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audio streams are not always necessary
@ -113,17 +110,15 @@ public class YoutubeStreamExtractorControversialTest {
assertTrue(streams.size() > 0);
}
@Ignore
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(!extractor.getSubtitlesDefault().isEmpty());
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Ignore
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty());
assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
}

39
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java → extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -53,7 +55,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
.getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
extractor.fetchPage();
}
@ -81,8 +83,8 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetFullLinksInDescriptlion() throws ParsingException {
assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt"));
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("http://adele.com"));
assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi..."));
}
@ -95,7 +97,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
assertEquals(366, extractor.getLength());
}
@Test
@ -111,7 +113,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetUploaderUrl() throws ParsingException {
assertTrue(extractor.getUploaderUrl().length() > 0);
assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
}
@Test
@ -231,4 +233,29 @@ public class YoutubeStreamExtractorDefaultTest {
assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
}
}
public static class FramesTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ");
extractor.fetchPage();
}
@Test
public void testGetFrames() throws ExtractionException {
final List<Frameset> frames = extractor.getFrames();
assertNotNull(frames);
assertFalse(frames.isEmpty());
for (final Frameset f : frames) {
for (final String url : f.getUrls()) {
ExtractorAsserts.assertIsValidUrl(url);
ExtractorAsserts.assertIsSecureUrl(url);
}
}
}
}
}

138
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java Normal file
View File

@ -0,0 +1,138 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorLivestreamTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY");
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetTitle() throws ParsingException {
assertFalse(extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(0, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
long count = extractor.getViewCount();
assertTrue(Long.toString(count), count >= 7148995);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
}
}
@Test
public void testStreamType() throws ParsingException {
assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
}
@Test
public void testGetDashMpd() throws ParsingException {
// we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty());
}
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
}

18
extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java Normal file
View File

@ -0,0 +1,18 @@
package org.schabi.newpipe.extractor.utils;
import com.grack.nanojson.JsonParserException;
import org.junit.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.junit.Assert.assertEquals;
public class UtilsTest {
@Test
public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
assertEquals(10, Utils.mixedNumberWordToLong("10"));
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
}
}

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

3
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,5 @@
#Fri Jan 18 11:51:40 CET 2019
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip

18
gradlew vendored
View File

@ -1,5 +1,21 @@
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
@ -28,7 +44,7 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"

18
gradlew.bat vendored
View File

@ -1,3 +1,19 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome