diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java index 2165002a8..a2fc7e006 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java @@ -7,15 +7,21 @@ import java.util.Map; import javax.annotation.Nonnull; public class DownloadResponse { + private final int responseCode; private final String responseBody; private final Map> responseHeaders; - public DownloadResponse(String responseBody, Map> headers) { + public DownloadResponse(int responseCode, String responseBody, Map> headers) { super(); + this.responseCode = responseCode; this.responseBody = responseBody; this.responseHeaders = headers; } + public int getResponseCode() { + return responseCode; + } + public String getResponseBody() { return responseBody; } @@ -33,5 +39,4 @@ public class DownloadResponse { else return cookies; } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java index 335e8a93e..f3526fcec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java @@ -60,6 +60,11 @@ public interface Downloader { */ String download(String siteUrl) throws IOException, ReCaptchaException; + DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException; + + DownloadResponse get(String siteUrl, Localization localization) + throws IOException, ReCaptchaException; + DownloadResponse get(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java index 5f0eaee44..09c2a1c05 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java @@ -21,7 +21,14 @@ package org.schabi.newpipe.extractor.exceptions; */ public class ReCaptchaException extends ExtractionException { - public ReCaptchaException(String message) { + private String url; + + public ReCaptchaException(String message, String url) { super(message); + this.url = url; + } + + public String getUrl() { + return url; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index dcc6c20c0..118b59cc0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -24,7 +24,9 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getThumbnailUrl() { - return itemObject.getString("avatar_url", ""); + String avatarUrl = itemObject.getString("avatar_url", ""); + String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg"); + return avatarUrlBetterResolution; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 317ccfb34..96b7fcea7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -7,9 +7,12 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -21,40 +24,59 @@ import java.io.IOException; import java.net.URLEncoder; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { + private static final String HARDCODED_CLIENT_ID = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19 private static String clientId; private SoundcloudParsingHelper() { } - public static String clientId() throws ReCaptchaException, IOException, RegexException { + public static String clientId() throws ExtractionException, IOException { if (clientId != null && !clientId.isEmpty()) return clientId; Downloader dl = NewPipe.getDownloader(); - String response = dl.download("https://soundcloud.com"); - - Document doc = Jsoup.parse(response); - Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first(); - - final String clientIdPattern = ",client_id:\"(.*?)\""; - - try { - final HashMap headers = new HashMap<>(); - headers.put("Range", "bytes=0-16384"); - String js = dl.download(jsElement.attr("src"), headers); - - return clientId = Parser.matchGroup1(clientIdPattern, js); - } catch (IOException | RegexException ignored) { - // Ignore it and proceed to download the whole js file + clientId = HARDCODED_CLIENT_ID; + if (checkIfHardcodedClientIdIsValid(dl)) { + return clientId; } - String js = dl.download(jsElement.attr("src")); - return clientId = Parser.matchGroup1(clientIdPattern, js); + final DownloadResponse download = dl.get("https://soundcloud.com"); + String response = download.getResponseBody(); + final String clientIdPattern = ",client_id:\"(.*?)\""; + + Document doc = Jsoup.parse(response); + final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]"); + // The one containing the client id will likely be the last one + Collections.reverse(possibleScripts); + + final HashMap headers = new HashMap<>(); + headers.put("Range", "bytes=0-16384"); + + for (Element element : possibleScripts) { + final String srcUrl = element.attr("src"); + if (srcUrl != null && !srcUrl.isEmpty()) { + try { + return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers)); + } catch (RegexException ignored) { + // Ignore it and proceed to try searching other script + } + } + } + + // Officially give up + throw new ExtractionException("Couldn't extract client id"); + } + + static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException { + final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID; + // Should return 200 to indicate that the client id is valid, a 401 is returned otherwise. + return dl.head(apiUrl).getResponseCode() == 200; } public static String toDateString(String time) throws ParsingException { @@ -79,7 +101,7 @@ public class SoundcloudParsingHelper { * * See https://developers.soundcloud.com/docs/api/reference#resolve */ - public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException { + public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { String apiUrl = "https://api.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); @@ -113,7 +135,10 @@ public class SoundcloudParsingHelper { String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url=" + URLEncoder.encode(url, "UTF-8")); - return Parser.matchGroup1(",\"id\":(.*?),", response); + // handle playlists / sets different and get playlist id via uir field in JSON + if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/")) + return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); + return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index e6d74b200..2c27fa91d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -71,13 +71,15 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { final String thumbnailUrl = item.getThumbnailUrl(); if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue; - return thumbnailUrl; + String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg"); + return thumbnailUrlBetterResolution; } } catch (Exception ignored) { } } - return artworkUrl; + String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); + return artworkUrlBetterResolution; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java index 4a6b4f1e2..53123859b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -32,7 +32,10 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr // Over-engineering at its finest if (itemObject.isString(ARTWORK_URL_KEY)) { final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); - if (!artworkUrl.isEmpty()) return artworkUrl; + if (!artworkUrl.isEmpty()) { + String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); + return artworkUrlBetterResolution; + } } try { @@ -42,8 +45,11 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr // First look for track artwork url if (trackObject.isString(ARTWORK_URL_KEY)) { - final String url = trackObject.getString(ARTWORK_URL_KEY, ""); - if (!url.isEmpty()) return url; + String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, ""); + if (!artworkUrl.isEmpty()) { + String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); + return artworkUrlBetterResolution; + } } // Then look for track creator avatar url diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java index 8dbe287eb..900571202 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; @@ -48,10 +49,10 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); - } catch (IOException e) { - throw new ParsingException("Could not get client id", e); } catch (ReCaptchaException e) { throw new ParsingException("ReCaptcha required", e); + } catch (IOException | ExtractionException e) { + throw new ParsingException("Could not get client id", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 4d84b22f7..e4459178f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -95,6 +95,7 @@ public class SoundcloudService extends StreamingService { try { list.addKioskEntry(chartsFactory, h, "Top 50"); list.addKioskEntry(chartsFactory, h, "New & hot"); + list.setDefaultKiosk("New & hot"); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 005722e3e..f5860d835 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -57,7 +57,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() { - return track.getString("artwork_url", ""); + String artworkUrl = track.getString("artwork_url", ""); + if (artworkUrl.isEmpty()) { + artworkUrl = track.getObject("user").getString("avatar_url", ""); + } + String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); + return artworkUrlBetterResolution; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index 358f32da0..09455e193 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -52,7 +52,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public String getThumbnailUrl() { - return itemObject.getString("artwork_url"); + String artworkUrl = itemObject.getString("artwork_url", ""); + if (artworkUrl.isEmpty()) { + artworkUrl = itemObject.getObject("user").getString("avatar_url"); + } + String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); + return artworkUrlBetterResolution; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d48b57a6a..9641d3931 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.DonationLinkHelper; @@ -47,8 +49,9 @@ import java.util.ArrayList; @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { + /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; - private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; + private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en"; private Document doc; @@ -59,8 +62,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; - String pageContent = downloader.download(channelUrl); - doc = Jsoup.parse(pageContent, channelUrl); + final DownloadResponse response = downloader.get(channelUrl); + doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); } @Override @@ -72,7 +75,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getUrl() throws ParsingException { try { - return "https://www.youtube.com/channel/" + getId(); + return CHANNEL_URL_BASE + getId(); } catch (ParsingException e) { return super.getUrl(); } @@ -81,6 +84,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { + try { + return doc.select("meta[itemprop=\"channelId\"]").first().attr("content"); + } catch (Exception ignored) {} + + // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) try { Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); @@ -134,10 +142,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { + final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); if (el != null) { + String elTitle = el.attr("title"); try { - return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + return Utils.mixedNumberWordToLong(elTitle); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 9e0e975f7..a687c0504 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -5,6 +5,9 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /* * Created by Christian Schabesberger on 12.02.17. * @@ -53,8 +56,26 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getUrl() throws ParsingException { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .attr("abs:href"); + try { + String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() + .attr("abs:data-href"); + + Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); + Matcher match = channelIdPattern.matcher(buttonTrackingUrl); + + if (match.matches()) { + return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); + } + } catch(Exception ignored) {} + + // fallback method for channels without "Subscribe" button (or just in case yt changes things) + // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail + try { + return el.select("a[class*=\"yt-uix-tile-link\"]").first() + .attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Could not get channel url", e); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 1d447bd55..a118b44ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -38,7 +38,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract try { return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); } catch (Exception e) { - throw new ParsingException("Could not get author name", e); + return ""; } } @@ -95,7 +95,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract try { return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); } catch (Exception e) { - throw new ParsingException("Could not get author name", e); + return ""; } } @@ -104,7 +104,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract try { return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId"); } catch (Exception e) { - throw new ParsingException("Could not get author endpoint", e); + return ""; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 64517f907..4480b38af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -35,8 +36,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String pageContent = downloader.download(getUrl()); - doc = Jsoup.parse(pageContent, getUrl()); + final String url = getUrl(); + final DownloadResponse response = downloader.get(url); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); } @Override @@ -50,7 +52,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text(); } catch (Exception e) { - throw new ParsingException("Could not get playlist name"); + throw new ParsingException("Could not get playlist name", e); } } @@ -59,7 +61,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src"); } catch (Exception e) { - throw new ParsingException("Could not get playlist thumbnail"); + throw new ParsingException("Could not get playlist thumbnail", e); } } @@ -72,9 +74,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { try { - return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href"); + return YoutubeChannelExtractor.CHANNEL_URL_BASE + + doc.select("button[class*=\"yt-uix-subscription-button\"]") + .first().attr("data-channel-external-id"); } catch (Exception e) { - throw new ParsingException("Could not get playlist uploader name"); + throw new ParsingException("Could not get playlist uploader url", e); } } @@ -83,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); } catch (Exception e) { - throw new ParsingException("Could not get playlist uploader name"); + throw new ParsingException("Could not get playlist uploader name", e); } } @@ -92,7 +96,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src"); } catch (Exception e) { - throw new ParsingException("Could not get playlist uploader avatar"); + throw new ParsingException("Could not get playlist uploader avatar", e); } } @@ -248,6 +252,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { + // this url is not always in the form "/channel/..." + // sometimes Youtube provides urls in the from "/user/..." return getUploaderLink().attr("abs:href"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 2d084909b..8812391f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -49,10 +49,11 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getUrl() throws ParsingException { try { - final Element div = el.select("div[class=\"yt-lockup-meta\"]").first(); + final Element a = el.select("div[class=\"yt-lockup-meta\"]") + .select("ul[class=\"yt-lockup-meta-info\"]") + .select("li").select("a").first(); - if(div != null) { - final Element a = div.select("a").first(); + if(a != null) { return a.attr("abs:href"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 709e5f577..0a954607f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; @@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -52,13 +54,9 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String site; final String url = getUrl(); - //String url = builder.build().toString(); - //if we've been passed a valid language code, append it to the URL - site = downloader.download(url, getLocalization()); - - doc = Jsoup.parse(site, url); + final DownloadResponse response = downloader.get(url, getLocalization()); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4b21a06ae..fa866cd5b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Parser; @@ -30,11 +31,13 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /* * Created by Christian Schabesberger on 06.08.15. * - * Copyright (C) Christian Schabesberger 2018 + * Copyright (C) Christian Schabesberger 2019 * YoutubeStreamExtractor.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify @@ -64,12 +67,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - public class GemaException extends ContentNotAvailableException { - GemaException(String message) { - super(message); - } - } - public class SubtitlesException extends ContentNotAvailableException { SubtitlesException(String message, Throwable cause) { super(message, cause); @@ -83,6 +80,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject playerArgs; @Nonnull private final Map videoInfoPage = new HashMap<>(); + private JsonObject playerResponse; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -162,14 +160,54 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + // onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;" + // :00 is NOT recognized as a timestamp in description or comments. + // 0:00 is recognized in both description and comments. + // https://www.youtube.com/watch?v=4cccfDXu1vA + private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile( + "seekTo\\(" + + "(?:(\\d+)\\*3600\\+)?" // hours? + + "(\\d+)\\*60\\+" // minutes + + "(\\d+)" // seconds + + "\\)"); + + @SafeVarargs + private static T coalesce(T... args) { + for (T arg : args) { + if (arg != null) return arg; + } + throw new IllegalArgumentException("all arguments to coalesce() were null"); + } + private String parseHtmlAndGetFullLinks(String descriptionHtml) throws MalformedURLException, UnsupportedEncodingException, ParsingException { final Document description = Jsoup.parse(descriptionHtml, getUrl()); for(Element a : description.select("a")) { - final URL redirectLink = new URL( - a.attr("abs:href")); - final String queryString = redirectLink.getQuery(); - if(queryString != null) { + final String rawUrl = a.attr("abs:href"); + final URL redirectLink = new URL(rawUrl); + + final Matcher onClickTimestamp; + final String queryString; + if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick"))) + .find()) { + a.removeAttr("onclick"); + + String hours = coalesce(onClickTimestamp.group(1), "0"); + String minutes = onClickTimestamp.group(2); + String seconds = onClickTimestamp.group(3); + + int timestamp = 0; + timestamp += Integer.parseInt(hours) * 3600; + timestamp += Integer.parseInt(minutes) * 60; + timestamp += Integer.parseInt(seconds); + + String setTimestamp = "&t=" + timestamp; + + // Even after clicking https://youtu.be/...?t=6, + // getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=. + a.attr("href", getUrl() + setTimestamp); + + } else if((queryString = redirectLink.getQuery()) != null) { // if the query string is null we are not dealing with a redirect link, // so we don't need to override it. final String link = @@ -179,11 +217,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { // if link is null the a tag is a hashtag. // They refer to the youtube search. We do not handle them. a.text(link); + a.attr("href", link); } else if(redirectLink.toString().contains("https://www.youtube.com/")) { a.text(redirectLink.toString()); + a.attr("href", redirectLink.toString()); } } else if(redirectLink.toString().contains("https://www.youtube.com/")) { + descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString()); a.text(redirectLink.toString()); + a.attr("href", redirectLink.toString()); } } return description.select("body").first().html(); @@ -206,29 +248,26 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getLength() throws ParsingException { assertPageFetched(); - if(playerArgs != null) { - try { - long returnValue = Long.parseLong(playerArgs.get("length_seconds") + ""); - if (returnValue >= 0) return returnValue; - } catch (Exception ignored) { - // Try other method... - } - } - String lengthString = videoInfoPage.get("length_seconds"); + // try getting duration from playerargs try { - return Long.parseLong(lengthString); - } catch (Exception ignored) { - // Try other method... - } - - // TODO: 25.11.17 Implement a way to get the length for age restricted videos #44 - try { - // Fallback to HTML method - return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first() - .attr("aria-valuemax")); + String durationMs = playerResponse + .getObject("streamingData") + .getArray("formats") + .getObject(0) + .getString("approxDurationMs"); + return Long.parseLong(durationMs)/1000; } catch (Exception e) { - throw new ParsingException("Could not get video length", e); + } + + //try getting value from age gated video + try { + String duration = playerResponse + .getObject("videoDetails") + .getString("lengthSeconds"); + return Long.parseLong(duration); + } catch (Exception e) { + throw new ParsingException("Every methode to get the duration has failed: ", e); } } @@ -385,31 +424,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getHlsUrl() throws ParsingException { assertPageFetched(); - try { - String hlsvp = ""; - if (playerArgs != null) { - if( playerArgs.isString("hlsvp") ) { - hlsvp = playerArgs.getString("hlsvp", ""); - }else { - hlsvp = JsonParser.object() - .from(playerArgs.getString("player_response", "{}")) - .getObject("streamingData", new JsonObject()) - .getString("hlsManifestUrl", ""); - } - } - return hlsvp; + try { + return playerResponse.getObject("streamingData").getString("hlsManifestUrl"); } catch (Exception e) { - throw new ParsingException("Could not get hls manifest url", e); + if (playerArgs != null && playerArgs.isString("hlsvp")) { + return playerArgs.getString("hlsvp"); + } else { + throw new ParsingException("Could not get hls manifest url", e); + } } } @Override - public List getAudioStreams() throws IOException, ExtractionException { + public List getAudioStreams() throws ExtractionException { assertPageFetched(); List audioStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { ItagItem itag = entry.getValue(); AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate); @@ -425,11 +457,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() throws ExtractionException { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); @@ -449,7 +481,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true); @@ -486,7 +518,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || - playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) { + (!playerResponse.getObject("streamingData").has(FORMATS)))) { return StreamType.LIVE_STREAM; } } catch (Exception e) { @@ -505,7 +537,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (watch.size() < 1) { return null;// prevent the snackbar notification "report error" on age-restricted videos } - + collector.commit(extractVideoPreviewInfo(watch.first().select("li").first())); return collector.getItems().get(0); } catch (Exception e) { @@ -538,21 +570,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); StringBuilder errorReason; + Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - if (errorMessage == null || errorMessage.isEmpty()) { + if (errorElement == null) { errorReason = null; - } else if (errorMessage.contains("GEMA")) { - // Gema sometimes blocks youtube music content in germany: - // https://www.gema.de/en/ - // Detailed description: - // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 - errorReason = new StringBuilder("GEMA"); } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + String errorMessage = errorElement.text(); + if (errorMessage == null || errorMessage.isEmpty()) { + errorReason = null; + } else { + errorReason = new StringBuilder(errorMessage); + errorReason.append(" "); + errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + } } return errorReason != null ? errorReason.toString() : null; @@ -562,8 +593,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Fetch page //////////////////////////////////////////////////////////////////////////*/ - private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - private static final String ADAPTIVE_FMTS = "adaptive_fmts"; + private static final String FORMATS = "formats"; + private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; @@ -571,7 +602,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX = - "(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; + "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX = @@ -581,22 +612,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String pageHtml = null; - private String getPageHtml(Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; - if (pageHtml == null) { - pageHtml = downloader.download(verifiedUrl); - } - return pageHtml; - } - @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String pageContent = getPageHtml(downloader); - doc = Jsoup.parse(pageContent, getUrl()); + final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; + final DownloadResponse response = downloader.get(verifiedUrl); + pageHtml = response.getResponseBody(); + doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response); final String playerUrl; // Check if the video is age restricted - if (pageContent.contains(" getAvailableSubtitlesInfo() throws SubtitlesException { // If the video is age restricted getPlayerConfig will fail if(isAgeRestricted) return Collections.emptyList(); - final JsonObject playerConfig; - try { - playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader())); - } catch (IOException | ExtractionException e) { - throw new SubtitlesException("Unable to download player configs", e); - } - final String playerResponse = playerConfig.getObject("args", new JsonObject()) - .getString("player_response"); - final JsonObject captions; - try { - if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) { - // Captions does not exist - return Collections.emptyList(); - } - captions = JsonParser.object().from(playerResponse).getObject("captions"); - } catch (JsonParserException e) { - throw new SubtitlesException("Unable to parse subtitles listing", e); + if (!playerResponse.has("captions")) { + // Captions does not exist + return Collections.emptyList(); } + captions = playerResponse.getObject("captions"); final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); @@ -849,39 +891,36 @@ public class YoutubeStreamExtractor extends StreamExtractor { "&sts=" + sts + "&ps=default&gl=US&hl=en"; } - private Map getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { + private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { Map urlAndItags = new LinkedHashMap<>(); - - String encodedUrlMap = ""; - if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) { - encodedUrlMap = playerArgs.getString(encodedUrlMapKey, ""); - } else if (videoInfoPage.containsKey(encodedUrlMapKey)) { - encodedUrlMap = videoInfoPage.get(encodedUrlMapKey); + JsonObject streamingData = playerResponse.getObject("streamingData"); + if (!streamingData.has(streamingDataKey)) { + return urlAndItags; } - for (String url_data_str : encodedUrlMap.split(",")) { - try { - // This loop iterates through multiple streams, therefore tags - // is related to one and the same stream at a time. - Map tags = Parser.compatParseMap( - org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); + JsonArray formats = streamingData.getArray(streamingDataKey); + for (int i = 0; i != formats.size(); ++i) { + JsonObject formatData = formats.getObject(i); + int itag = formatData.getInt("itag"); - int itag = Integer.parseInt(tags.get("itag")); - - if (ItagItem.isSupported(itag)) { + if (ItagItem.isSupported(itag)) { + try { ItagItem itagItem = ItagItem.getItag(itag); if (itagItem.itagType == itagTypeWanted) { - String streamUrl = tags.get("url"); - // if video has a signature: decrypt it and add it to the url - if (tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); + String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // this url has an encrypted signature + Map cipher = Parser.compatParseMap(formatData.getString("cipher")); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode); } + urlAndItags.put(streamUrl, itagItem); } + } catch (UnsupportedEncodingException ignored) { + } - } catch (DecryptException e) { - throw e; - } catch (Exception ignored) { } } @@ -954,4 +993,61 @@ public class YoutubeStreamExtractor extends StreamExtractor { } }; } + + @Nonnull + @Override + public List getFrames() throws ExtractionException { + try { + final String script = doc.select("#player-api").first().siblingElements().select("script").html(); + int p = script.indexOf("ytplayer.config"); + if (p == -1) { + return Collections.emptyList(); + } + p = script.indexOf('{', p); + int e = script.indexOf("ytplayer.load", p); + if (e == -1) { + return Collections.emptyList(); + } + JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); + final String resp = jo.getObject("args").getString("player_response"); + jo = JsonParser.object().from(resp); + final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); + final String url = spec[0]; + final ArrayList result = new ArrayList<>(spec.length - 1); + for (int i = 1; i < spec.length; ++i) { + final String[] parts = spec[i].split("#"); + if (parts.length != 8) { + continue; + } + final int frameWidth = Integer.parseInt(parts[0]); + final int frameHeight = Integer.parseInt(parts[1]); + final int totalCount = Integer.parseInt(parts[2]); + final int framesPerPageX = Integer.parseInt(parts[3]); + final int framesPerPageY = Integer.parseInt(parts[4]); + final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; + final List urls; + if (baseUrl.contains("$M")) { + final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); + urls = new ArrayList<>(totalPages); + for (int j = 0; j < totalPages; j++) { + urls.add(baseUrl.replace("$M", String.valueOf(j))); + } + } else { + urls = Collections.singletonList(baseUrl); + } + result.add(new Frameset( + urls, + frameWidth, + frameHeight, + totalCount, + framesPerPageX, + framesPerPageY + )); + } + result.trimToSize(); + return result; + } catch (Exception e) { + throw new ExtractionException(e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 1aca59399..5bfeaa38e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -61,7 +61,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUrl() throws ParsingException { try { - Element el = item.select("div[class*=\"yt-lockup-video\"").first(); + Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); Element dl = el.select("h3").first().select("a").first(); return dl.attr("abs:href"); } catch (Exception e) { @@ -72,7 +72,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { try { - Element el = item.select("div[class*=\"yt-lockup-video\"").first(); + Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); Element dl = el.select("h3").first().select("a").first(); return dl.text(); } catch (Exception e) { @@ -107,6 +107,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderUrl() throws ParsingException { + // this url is not always in the form "/channel/..." + // sometimes Youtube provides urls in the from "/user/..." try { try { return item.select("div[class=\"yt-lockup-byline\"]").first() @@ -119,7 +121,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { .text().split(" - ")[0]; } catch (Exception e) { System.out.println(item.html()); - throw new ParsingException("Could not get uploader", e); + throw new ParsingException("Could not get uploader url", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index f6b6b8bd9..55a9c7c1f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -63,16 +63,10 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { String title = outline.attr("title"); String xmlUrl = outline.attr("abs:xmlUrl"); - if (title.isEmpty() || xmlUrl.isEmpty()) { - throw new InvalidSourceException("document has invalid entries"); - } - try { String id = Parser.matchGroup1(ID_PATTERN, xmlUrl); result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title)); - } catch (Parser.RegexException e) { - throw new InvalidSourceException("document has invalid entries", e); - } + } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ } } return result; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index b065aa630..dc7cc7e69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -24,12 +24,14 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Localization; @@ -56,8 +58,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor { url += "?gl=" + contentCountry; } - String pageContent = downloader.download(url); - doc = Jsoup.parse(pageContent, url); + final DownloadResponse response = downloader.get(url); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); } @Override @@ -126,6 +128,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } private Element getUploaderLink() { + // this url is not always in the form "/channel/..." + // sometimes Youtube provides urls in the from "/user/..." Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); return uploaderEl.select("a").first(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index e3522b313..5a2e687c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -46,7 +46,8 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); - if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) { + if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || + YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) { throw new ParsingException("the URL given is not a Youtube-URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 335bc5bf6..120275caa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,7 +1,11 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import java.net.URL; @@ -30,40 +34,42 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - private static boolean isHTTP(URL url) { - // make sure its http or https - String protocol = url.getProtocol(); - if (!protocol.equals("http") && !protocol.equals("https")) { - return false; + private static final String[] RECAPTCHA_DETECTION_SELECTORS = { + "form[action*=\"/das_captcha\"]", + "input[name*=\"action_recaptcha_verify\"]" + }; + + public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException { + final Document document = Jsoup.parse(response.getResponseBody(), url); + + for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) { + if (!document.select(detectionSelector).isEmpty()) { + throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url); + } } - boolean usesDefaultPort = url.getPort() == url.getDefaultPort(); - boolean setsNoPort = url.getPort() == -1; - - return setsNoPort || usesDefaultPort; + return document; } public static boolean isYoutubeURL(URL url) { - // make sure its http or https - if (!isHTTP(url)) - return false; - - // make sure its a known youtube url String host = url.getHost(); return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") - || host.equalsIgnoreCase("m.youtube.com"); + || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com"); } - public static boolean isYoutubeALikeURL(URL url) { - // make sure its http or https - if (!isHTTP(url)) - return false; - - // make sure its a known youtube url + public static boolean isYoutubeServiceURL(URL url) { String host = url.getHost(); - return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") - || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("www.youtube-nocookie.com") - || host.equalsIgnoreCase("youtu.be") || host.equalsIgnoreCase("hooktube.com"); + return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be"); + } + + public static boolean isHooktubeURL(URL url) { + String host = url.getHost(); + return host.equalsIgnoreCase("hooktube.com"); + } + + public static boolean isInvidioURL(URL url) { + String host = url.getHost(); + return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org"); } public static long parseDurationString(String input) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 008aeb933..62a0e7375 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -25,10 +25,16 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { try { URL urlObj = Utils.stringToURL(url); - if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) { + if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) + || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } + String path = urlObj.getPath(); + if (!path.equals("/watch" ) && !path.equals("/playlist")) { + throw new ParsingException("the url given is neither a video nor a playlist URL"); + } + String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index d9d9e93a0..e3ac4d520 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -60,7 +60,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { URI uri = new URI(urlString); String scheme = uri.getScheme(); - if (scheme != null && scheme.equals("vnd.youtube")) { + if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { String schemeSpecificPart = uri.getSchemeSpecificPart(); if (schemeSpecificPart.startsWith("//")) { urlString = "https:" + schemeSpecificPart; @@ -85,7 +85,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { path = path.substring(1); } - if (!YoutubeParsingHelper.isYoutubeALikeURL(url)) { + if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) || + YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) || + YoutubeParsingHelper.isInvidioURL(url))) { if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { throw new FoundAdException("Error found ad: " + urlString); } @@ -112,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { case "YOUTUBE.COM": case "WWW.YOUTUBE.COM": - case "M.YOUTUBE.COM": { + case "M.YOUTUBE.COM": + case "MUSIC.YOUTUBE.COM": { if (path.equals("attribution_link")) { String uQueryValue = Utils.getQueryValue(url, "u"); @@ -147,6 +150,34 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } case "HOOKTUBE.COM": { + if (path.startsWith("v/")) { + String id = path.substring("v/".length()); + + return assertIsID(id); + } + if (path.startsWith("watch/")) { + String id = path.substring("watch/".length()); + + return assertIsID(id); + } + // there is no break-statement here on purpose so the next code-block gets also run for hooktube + } + + case "WWW.INVIDIO.US": + case "DEV.INVIDIO.US": + case "INVIDIO.US": + case "INVIDIOUS.SNOPYTA.ORG": + case "DE.INVIDIOUS.SNOPYTA.ORG": + case "FI.INVIDIOUS.SNOPYTA.ORG": + case "VID.WXZM.SX": + case "INVIDIOUS.KABI.TK": + case "INVIDIOU.SH": + case "WWW.INVIDIOU.SH": + case "NO.INVIDIOU.SH": + case "INVIDIOUS.ENKIRTON.NET": + case "TUBE.POAL.CO": + case "INVIDIOUS.13AD.DE": + case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { @@ -158,19 +189,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } - if (path.startsWith("v/")) { - String id = path.substring("v/".length()); - return assertIsID(id); - } - if (path.startsWith("watch/")) { - String id = path.substring("watch/".length()); - - return assertIsID(id); - } + break; } - - break; } throw new ParsingException("Error no suitable url: " + urlString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 253e9cd8a..799681128 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -48,6 +48,6 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { } String urlPath = urlObj.getPath(); - return YoutubeParsingHelper.isYoutubeURL(urlObj) && urlPath.equals("/feed/trending"); + return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java new file mode 100644 index 000000000..2d4010dd9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java @@ -0,0 +1,66 @@ +package org.schabi.newpipe.extractor.stream; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; + +public final class Frameset { + + private List urls; + private int frameWidth; + private int frameHeight; + private int totalCount; + private int framesPerPageX; + private int framesPerPageY; + + public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { + this.urls = urls; + this.totalCount = totalCount; + this.frameWidth = frameWidth; + this.frameHeight = frameHeight; + this.framesPerPageX = framesPerPageX; + this.framesPerPageY = framesPerPageY; + } + + /** + * @return list of urls to images with frames + */ + public List getUrls() { + return urls; + } + + /** + * @return total count of frames + */ + public int getTotalCount() { + return totalCount; + } + + /** + * @return maximum frames count by x + */ + public int getFramesPerPageX() { + return framesPerPageX; + } + + /** + * @return maximum frames count by y + */ + public int getFramesPerPageY() { + return framesPerPageY; + } + + /** + * @return width of a one frame, in pixels + */ + public int getFrameWidth() { + return frameWidth; + } + + /** + * @return height of a one frame, in pixels + */ + public int getFrameHeight() { + return frameHeight; + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 3b9f06532..46ffe8a8a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -30,7 +30,10 @@ import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -256,7 +259,18 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException; /** - * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block) + * Should return a list of Frameset object that contains preview of stream frames + * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream + * @throws IOException + * @throws ExtractionException + */ + @Nonnull + public List getFrames() throws IOException, ExtractionException { + return Collections.emptyList(); + } + + /** + * Should analyse the webpage's document and extracts any error message there might be. * * @return Error message; null if there is no error message. */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 8a994cbde..9c1a8a16a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -123,8 +123,6 @@ public class DashMpdParser { dashDoc = downloader.download(streamInfo.getDashMpdUrl()); } catch (IOException ioe) { throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe); - } catch (ReCaptchaException e) { - throw new ReCaptchaException("reCaptcha Challenge needed"); } try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fde334fa7..1f8da13b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -27,6 +27,38 @@ public class Utils { return toRemove.replaceAll("\\D+", ""); } + /** + *

Convert a mixed number word to a long.

+ *

Examples:

+ *
    + *
  • 123 -> 123
    • + *
  • 1.23K -> 1230
    • + *
  • 1.23M -> 1230000
    • + *
+ * @param numberWord string to be converted to a long + * @return a long + * @throws NumberFormatException + * @throws ParsingException + */ + public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException { + String multiplier = ""; + try { + multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2); + } catch(ParsingException ignored) {} + double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord) + .replace(",", ".")); + switch (multiplier.toUpperCase()) { + case "K": + return (long) (count * 1e3); + case "M": + return (long) (count * 1e6); + case "B": + return (long) (count * 1e9); + default: + return (long) (count); + } + } + /** * Check if the url matches the pattern. * @@ -120,6 +152,19 @@ public class Utils { throw e; } } + + public static boolean isHTTP(URL url) { + // make sure its http or https + String protocol = url.getProtocol(); + if (!protocol.equals("http") && !protocol.equals("https")) { + return false; + } + + boolean usesDefaultPort = url.getPort() == url.getDefaultPort(); + boolean setsNoPort = url.getPort() == -1; + + return setsNoPort || usesDefaultPort; + } public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { diff --git a/extractor/src/test/java/org/schabi/newpipe/Downloader.java b/extractor/src/test/java/org/schabi/newpipe/Downloader.java index c980e1e9a..3091c74bb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/Downloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/Downloader.java @@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Localization; +import static java.util.Collections.singletonList; + /* * Created by Christian Schabesberger on 28.01.16. * @@ -129,7 +131,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { * request See : https://github.com/rg3/youtube-dl/issues/5138 */ if (con.getResponseCode() == 429) { - throw new ReCaptchaException("reCaptcha Challenge requested"); + throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString()); } throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e); @@ -172,6 +174,36 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { return dl(con); } + @Override + public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException { + final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection(); + + try { + con.setRequestMethod("HEAD"); + setDefaults(con); + } catch (Exception e) { + /* + * HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge + * request See : https://github.com/rg3/youtube-dl/issues/5138 + */ + if (con.getResponseCode() == 429) { + throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString()); + } + + throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e); + } + + return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields()); + } + + @Override + public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException { + final Map> requestHeaders = new HashMap<>(); + requestHeaders.put("Accept-Language", singletonList(localization.getLanguage())); + + return get(siteUrl, new DownloadRequest(null, requestHeaders)); + } + @Override public DownloadResponse get(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException { @@ -183,7 +215,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { } } String responseBody = dl(con); - return new DownloadResponse(responseBody, con.getHeaderFields()); + return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields()); } @Override @@ -219,6 +251,6 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { sb.append(inputLine); } } - return new DownloadResponse(sb.toString(), con.getHeaderFields()); + return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index 8f075ed90..54d81f160 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -36,7 +36,7 @@ public class MediaCCCOggTest { @Test public void getAudioStreamsContainOgg() throws Exception { for(AudioStream stream : extractor.getAudioStreams()) { - System.out.println(stream.getFormat()); + assertEquals("OGG", stream.getFormat().toString()); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index b7c39ca28..f62771d69 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -25,7 +25,7 @@ import org.schabi.newpipe.extractor.utils.Localization; * Test for {@link PeertubeChannelExtractor} */ public class PeertubeChannelExtractorTest { - public static class LilUzi implements BaseChannelExtractorTest { + public static class KDE implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -34,7 +34,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); extractor.fetchPage(); } @@ -49,22 +49,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("Noureddine HADDAG", extractor.getName()); + assertEquals("The KDE Community", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("root@tube.openalgeria.org", extractor.getId()); + assertEquals("kde", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -112,7 +112,7 @@ public class PeertubeChannelExtractorTest { } } - public static class DubMatix implements BaseChannelExtractorTest { + public static class Booteille implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -121,7 +121,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch"); + .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); extractor.fetchPage(); } @@ -146,22 +146,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("France Inter", extractor.getName()); + assertEquals("booteille", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("franceinter@tube.kdy.ch", extractor.getId()); + assertEquals("booteille", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -205,7 +205,7 @@ public class PeertubeChannelExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 33ab98a30..79e52e6b4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -32,7 +32,7 @@ public class PeertubeStreamExtractorDefaultTest { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); // setting instance might break test when running in parallel PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); extractor.fetchPage(); } @@ -44,39 +44,39 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "The Internet's Own Boy"); + assertEquals(extractor.getName(), "Power Corrupts the Best"); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "The story of programming prodigy and information activist Aaron Swartz, who took his own life at the age of 26."); + assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "root"); + assertEquals(extractor.getUploaderName(), "Rowsedower"); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 6299); + assertEquals(extractor.getLength(), 269); } @Test public void testGetViewCount() throws ParsingException { assertTrue(Long.toString(extractor.getViewCount()), - extractor.getViewCount() > 700); + extractor.getViewCount() > 10); } @Test public void testGetUploadDate() throws ParsingException { - assertEquals("2017-10-17", extractor.getUploadDate()); + assertEquals("2018-09-30", extractor.getUploadDate()); } @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@peertube2.cpy.re", extractor.getUploaderUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index 82f30c6ea..b9b9535e8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -25,23 +25,21 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); // setting instance might break test when running in parallel PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); - extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy"); + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde"); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet%27s+own+boy&start=12&count=12", extractor.getNextPageUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl()); } @Test public void testResultList_FirstElement() { InfoItem firstInfoItem = itemsPage.getItems().get(0); - - // THe channel should be the first item - assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName()); - assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl()); + + assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); } @Test @@ -83,11 +81,11 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @Test public void testId() throws Exception { - assertEquals("internet's own boy", extractor.getId()); + assertEquals("kde", extractor.getId()); } @Test public void testName() { - assertEquals("internet's own boy", extractor.getName()); + assertEquals("kde", extractor.getName()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index 8e93f5241..dea7a8a9c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -36,7 +36,6 @@ public class SoundcloudChartsExtractorTest { assertNotNull(NewPipe.getDownloader()); } - @Ignore @Test public void testGetName() throws Exception { assertEquals(extractor.getName(), "Top 50"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index b1771f068..435ae446c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -1,18 +1,24 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.utils.Localization; +import static org.junit.Assert.*; + public class SoundcloudParsingHelperTest { @BeforeClass public static void setUp() { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); } + @Test + public void assertThatHardcodedClientIdIsValid() throws Exception { + assertTrue("Hardcoded client id is not valid anymore", + SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(Downloader.getInstance())); + } + @Test public void resolveUrlWithEmbedPlayerTest() throws Exception { Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index c014b3417..ec020109e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractorTest { } } - public static class RandomHouseDanceMusic implements BasePlaylistExtractorTest { + public static class RandomHouseMusic implements BasePlaylistExtractorTest { private static SoundcloudPlaylistExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (SoundcloudPlaylistExtractor) SoundCloud - .getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2"); + .getPlaylistExtractor("https://soundcloud.com/micky96/sets/house"); extractor.fetchPage(); } @@ -141,22 +142,22 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testName() { - assertEquals("House, Electro , Dance Music 2", extractor.getName()); + assertEquals("House", extractor.getName()); } @Test public void testId() { - assertEquals("310980722", extractor.getId()); + assertEquals("123062856", extractor.getId()); } @Test public void testUrl() throws Exception { - assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl()); + assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getUrl()); } @Test public void testOriginalUrl() throws Exception { - assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl()); + assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -182,7 +183,7 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore + @Ignore("not implemented") @Test public void testBannerUrl() { assertIsSecureUrl(extractor.getBannerUrl()); @@ -192,12 +193,12 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader")); + assertThat(uploaderUrl, CoreMatchers.containsString("micky96")); } @Test public void testUploaderName() { - assertEquals("Gosu", extractor.getUploaderName()); + assertEquals("_mickyyy", extractor.getUploaderName()); } @Test @@ -226,6 +227,7 @@ public class SoundcloudPlaylistExtractorTest { // Additional Testing //////////////////////////////////////////////////////////////////////////*/ + @Ignore @Test public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); @@ -265,11 +267,14 @@ public class SoundcloudPlaylistExtractorTest { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Ignore @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); } + //TODO: FUCK THIS: This triggers a 500 at sever + @Ignore @Test public void testMoreRelatedItems() throws Exception { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); @@ -284,6 +289,7 @@ public class SoundcloudPlaylistExtractorTest { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ + @Ignore @Test public void testThumbnailUrl() { assertIsSecureUrl(extractor.getThumbnailUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 3970efde8..c8a2a841c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -39,27 +39,27 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetValidTimeStamp() throws IOException, ExtractionException { StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); - assertEquals(extractor.getTimeStamp() + "", "69"); + assertEquals("69", extractor.getTimeStamp() + ""); } @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Do What I Want [Produced By Maaly Raw + Don Cannon]"); + assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "The Perfect LUV Tape®️"); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "LIL UZI VERT"); + assertEquals("LIL UZI VERT", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 175); + assertEquals(175, extractor.getLength()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 2fb6f2d99..b67041bbd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -109,6 +109,7 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); + assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6); } } @@ -199,6 +200,7 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); + assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6); } } @@ -394,6 +396,100 @@ public class YoutubeChannelExtractorTest { } } + // this channel has no "Subscribe" button + public static class EminemVEVO implements BaseChannelExtractorTest { + private static YoutubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeChannelExtractor) YouTube + .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("EminemVEVO", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, YouTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, YouTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws Exception { + final String description = extractor.getDescription(); + assertTrue(description, description.contains("Eminem on Vevo")); + } + + @Test + public void testAvatarUrl() throws Exception { + String avatarUrl = extractor.getAvatarUrl(); + assertIsSecureUrl(avatarUrl); + assertTrue(avatarUrl, avatarUrl.contains("yt3")); + } + + @Test + public void testBannerUrl() throws Exception { + String bannerUrl = extractor.getBannerUrl(); + assertIsSecureUrl(bannerUrl); + assertTrue(bannerUrl, bannerUrl.contains("yt3")); + } + + @Test + public void testFeedUrl() throws Exception { + assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws Exception { + // there is no "Subscribe" button + long subscribers = extractor.getSubscriberCount(); + assertEquals("Wrong subscriber count", -1, subscribers); + } + } + + + public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -485,8 +581,9 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); + long subscribers = extractor.getSubscriberCount(); + assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50); } } -}; +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index ba858bbc1..378308773 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -37,6 +37,12 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + + assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh")); + assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos")); + + assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); + assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); } @Test @@ -53,5 +59,11 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); + + assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId()); + assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId()); + + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 127eef7ab..6d22e4d66 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -27,18 +27,18 @@ public class YoutubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor("https://www.youtube.com/watch?v=ehn8mJ8vvsI"); + .getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o"); } @Test public void testGetComments() throws IOException, ExtractionException { - boolean result = false; + boolean result; InfoItemsPage comments = extractor.getInitialPage(); - result = findInComments(comments, "Tsuki blyat"); + result = findInComments(comments, "s1ck m3m3"); while (comments.hasNextPage() && !result) { comments = extractor.getPage(comments.getNextPageUrl()); - result = findInComments(comments, "Tsuki blyat"); + result = findInComments(comments, "s1ck m3m3"); } assertTrue(result); @@ -47,14 +47,14 @@ public class YoutubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=ehn8mJ8vvsI"); - assertTrue("the dark side of YouTube...".equals(commentsInfo.getName())); - result = findInComments(commentsInfo.getRelatedItems(), "Tsuki blyat"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o"); + assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName())); + result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); String nextPage = commentsInfo.getNextPageUrl(); while (!StringUtil.isBlank(nextPage) && !result) { InfoItemsPage moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); - result = findInComments(moreItems.getItems(), "Tsuki blyat"); + result = findInComments(moreItems.getItems(), "s1ck m3m3"); nextPage = moreItems.getNextPageUrl(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 9cfd6c00e..9f3c40490 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -100,7 +100,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderUrl() throws Exception { - assertTrue(extractor.getUploaderUrl().contains("youtube.com")); + assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); } @Test @@ -185,8 +185,8 @@ public class YoutubePlaylistExtractorTest { public void testMoreRelatedItems() throws Exception { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); - // Test for 2 more levels + // test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors()); @@ -214,7 +214,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderUrl() throws Exception { - assertTrue(extractor.getUploaderUrl().contains("youtube.com")); + assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java new file mode 100644 index 000000000..4e2d148cd --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -0,0 +1,107 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +import static org.junit.Assert.*; + +/** + * Test for {@link YoutubePlaylistLinkHandlerFactory} + */ +public class YoutubePlaylistLinkHandlerFactoryTest { + private static YoutubePlaylistLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = YoutubePlaylistLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test(expected = IllegalArgumentException.class) + public void getIdWithNullAsUrl() throws ParsingException { + linkHandler.fromId(null); + } + + @Test + public void getIdfromYt() throws Exception { + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId()); + } + + @Test + public void testAcceptYtUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); + assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); + } + + @Test + public void testDeniesInvalidYtUrl() throws ParsingException { + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("www.youtube.com:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("youtube . com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + } + + @Test + public void testAcceptInvidioUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); + assertTrue(linkHandler.acceptUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + } + + @Test + public void testDeniesInvalidInvidioUrl() throws ParsingException { + assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("invidio.us:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("invidio . us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + } + + @Test + public void testGetInvidioIdfromUrl() throws ParsingException { + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index f06ad319d..154dcdd26 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); + assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); } @Test @@ -98,13 +99,14 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); - - assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); + assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI")); + assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); } @Test public void testAcceptHookUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU")); assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); assertTrue(linkHandler.acceptUrl("hooktube.com/watch/3msbfr6pBNE")); @@ -115,10 +117,31 @@ public class YoutubeStreamLinkHandlerFactoryTest { @Test public void testGetHookIdfromUrl() throws ParsingException { assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId()); + assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId()); assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } + + @Test + public void testAcceptInvidioUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE")); + assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); + assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE")); + } + + @Test + public void testGetInvidioIdfromUrl() throws ParsingException { + assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId()); + assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId()); + assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId()); + assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId()); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 8d2e6cfae..c5739b854 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -59,15 +59,38 @@ public class YoutubeSubscriptionExtractorTest { assertTrue(items.isEmpty()); } + @Test + public void testSubscriptionWithEmptyTitleInSource() throws Exception { + String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA"; + String source = "" + + "" + + ""; + + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); + assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1); + assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty()); + assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId)); + } + + @Test + public void testSubscriptionWithInvalidUrlInSource() throws Exception { + String source = "" + + "" + + "" + + "" + + "" + + ""; + + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); + assertTrue(items.isEmpty()); + } + @Test public void testInvalidSourceException() { List invalidList = Arrays.asList( "", "", "", - "", - "", "", null, "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28", @@ -78,11 +101,11 @@ public class YoutubeSubscriptionExtractorTest { if (invalidContent != null) { byte[] bytes = invalidContent.getBytes("UTF-8"); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); + fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); } else { subscriptionExtractor.fromInputStream(null); + fail("Extracting from null String didn't throw an exception"); } - - fail("didn't throw exception"); } catch (Exception e) { // System.out.println(" -> " + e); boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java index 46095b4da..ddb5550c9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java @@ -69,6 +69,10 @@ public class YoutubeTrendingLinkHandlerFactoryTest { assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending")); assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending")); + assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending")); assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia")); assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 14b94b510..1031ce241 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.search; +import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.InfoItem; @@ -53,6 +55,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl()); } + @Ignore @Test public void testOnlyContainChannels() { for(InfoItem item : itemsPage.getItems()) { @@ -61,4 +64,19 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } } + + @Test + public void testChannelUrl() { + for(InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { + ChannelInfoItem channel = (ChannelInfoItem) item; + + if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie + assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); + } else { + assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + } + } + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index f25b00197..bef1c6207 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -73,7 +73,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas assertTrue((firstInfoItem instanceof ChannelInfoItem) || (secondInfoItem instanceof ChannelInfoItem)); assertEquals("name", "PewDiePie", channelItem.getName()); - assertEquals("url","https://www.youtube.com/user/PewDiePie", channelItem.getUrl()); + assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java similarity index 94% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 8a91887af..6a24ec5f2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Ignore; @@ -71,10 +71,9 @@ public class YoutubeStreamExtractorAgeRestrictedTest { assertFalse(extractor.getUploaderName().isEmpty()); } - @Ignore // Currently there is no way get the length from restricted videos @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(1789, extractor.getLength()); } @Test @@ -97,8 +96,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest { assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } - // FIXME: 25.11.17 Are there no streams or are they not listed? - @Ignore @Test public void testGetAudioStreams() throws IOException, ExtractionException { // audio streams are not always necessary diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java similarity index 90% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index a300d6228..261d521c1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Ignore; @@ -71,10 +71,9 @@ public class YoutubeStreamExtractorControversialTest { assertFalse(extractor.getUploaderName().isEmpty()); } - @Ignore // Currently there is no way get the length from restricted videos @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(219, extractor.getLength()); } @Test @@ -97,8 +96,6 @@ public class YoutubeStreamExtractorControversialTest { assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } - // FIXME: 25.11.17 Are there no streams or are they not listed? - @Ignore @Test public void testGetAudioStreams() throws IOException, ExtractionException { // audio streams are not always necessary @@ -113,17 +110,15 @@ public class YoutubeStreamExtractorControversialTest { assertTrue(streams.size() > 0); } - @Ignore @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(!extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } - @Ignore @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java similarity index 87% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index f4d8a3540..a0e5050ab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -1,8 +1,9 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.List; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -53,7 +55,7 @@ public class YoutubeStreamExtractorDefaultTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw"); + .getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A"); extractor.fetchPage(); } @@ -81,8 +83,8 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetFullLinksInDescriptlion() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); + public void testGetFullLinksInDescription() throws ParsingException { + assertTrue(extractor.getDescription().contains("http://adele.com")); assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @@ -95,7 +97,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(366, extractor.getLength()); } @Test @@ -111,7 +113,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetUploaderUrl() throws ParsingException { - assertTrue(extractor.getUploaderUrl().length() > 0); + assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl()); } @Test @@ -231,4 +233,29 @@ public class YoutubeStreamExtractorDefaultTest { assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); } } + + public static class FramesTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ"); + extractor.fetchPage(); + } + + @Test + public void testGetFrames() throws ExtractionException { + final List frames = extractor.getFrames(); + assertNotNull(frames); + assertFalse(frames.isEmpty()); + for (final Frameset f : frames) { + for (final String url : f.getUrls()) { + ExtractorAsserts.assertIsValidUrl(url); + ExtractorAsserts.assertIsSecureUrl(url); + } + } + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java new file mode 100644 index 000000000..3e24b7e36 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -0,0 +1,138 @@ +package org.schabi.newpipe.extractor.services.youtube.stream; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeStreamExtractorLivestreamTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertFalse(extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } + + @Test + public void testGetFullLinksInDescription() throws ParsingException { + assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); + assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + + @Test + public void testGetLength() throws ParsingException { + assertEquals(0, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + long count = extractor.getViewCount(); + assertTrue(Long.toString(count), count >= 7148995); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws ExtractionException { + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testGetVideoStreams() throws ExtractionException { + for (VideoStream s : extractor.getVideoStreams()) { + assertIsSecureUrl(s.url); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.getFormatId()), + 0 <= s.getFormatId() && s.getFormatId() <= 0x100); + } + } + + @Test + public void testStreamType() throws ParsingException { + assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM); + } + + @Test + public void testGetDashMpd() throws ParsingException { + // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. + assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty()); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + Utils.printErrors(relatedVideos.getErrors()); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java new file mode 100644 index 000000000..578867445 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.utils; + +import com.grack.nanojson.JsonParserException; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import static org.junit.Assert.assertEquals; + +public class UtilsTest { + @Test + public void testMixedNumberWordToLong() throws JsonParserException, ParsingException { + assertEquals(10, Utils.mixedNumberWordToLong("10")); + assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0); + assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0); + assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0); + assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0); + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ed88a042a..5c2d1cf01 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ade732376..4b7e1f3d3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Fri Jan 18 11:51:40 CET 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip diff --git a/gradlew b/gradlew index cccdd3d51..8e25e6c19 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index e95643d6a..24467a141 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome