merged upstream/dev

extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java

@@ -7,15 +7,21 @@ import java.util.Map;
 import javax.annotation.Nonnull;
 
 public class DownloadResponse {
+    private final int responseCode;
     private final String responseBody;
     private final Map<String, List<String>> responseHeaders;
 
-    public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
+    public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
         super();
+        this.responseCode = responseCode;
         this.responseBody = responseBody;
         this.responseHeaders = headers;
     }
 
+    public int getResponseCode() {
+        return responseCode;
+    }
+
     public String getResponseBody() {
         return responseBody;
     }
@@ -33,5 +39,4 @@ public class DownloadResponse {
         else
             return cookies;
     }
-
 }

extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java

@@ -60,6 +60,11 @@ public interface Downloader {
      */
     String download(String siteUrl) throws IOException, ReCaptchaException;
 
+    DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
+
+    DownloadResponse get(String siteUrl, Localization localization)
+            throws IOException, ReCaptchaException;
+
     DownloadResponse get(String siteUrl, DownloadRequest request)
             throws IOException, ReCaptchaException;
 

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java

@@ -21,7 +21,14 @@ package org.schabi.newpipe.extractor.exceptions;
  */
 
 public class ReCaptchaException extends ExtractionException {
-    public ReCaptchaException(String message) {
+    private String url;
+
+    public ReCaptchaException(String message, String url) {
         super(message);
+        this.url = url;
+    }
+
+    public String getUrl() {
+        return url;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -24,7 +24,9 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
 
     @Override
     public String getThumbnailUrl() {
-        return itemObject.getString("avatar_url", "");
+        String avatarUrl = itemObject.getString("avatar_url", "");
+        String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
+        return avatarUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -7,9 +7,12 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@@ -21,40 +24,59 @@ import java.io.IOException;
 import java.net.URLEncoder;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudParsingHelper {
+    private static final String HARDCODED_CLIENT_ID = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19
     private static String clientId;
     
     private SoundcloudParsingHelper() {
     }
 
-    public static String clientId() throws ReCaptchaException, IOException, RegexException {
+    public static String clientId() throws ExtractionException, IOException {
         if (clientId != null && !clientId.isEmpty()) return clientId;
 
         Downloader dl = NewPipe.getDownloader();
-        String response = dl.download("https://soundcloud.com");
+        clientId = HARDCODED_CLIENT_ID;
-
+        if (checkIfHardcodedClientIdIsValid(dl)) {
-        Document doc = Jsoup.parse(response);
+            return clientId;
-        Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
-
-        final String clientIdPattern = ",client_id:\"(.*?)\"";
-
-        try {
-            final HashMap<String, String> headers = new HashMap<>();
-            headers.put("Range", "bytes=0-16384");
-            String js = dl.download(jsElement.attr("src"), headers);
-
-            return clientId = Parser.matchGroup1(clientIdPattern, js);
-        } catch (IOException | RegexException ignored) {
-            // Ignore it and proceed to download the whole js file
         }
 
-        String js = dl.download(jsElement.attr("src"));
+        final DownloadResponse download = dl.get("https://soundcloud.com");
-        return clientId = Parser.matchGroup1(clientIdPattern, js);
+        String response = download.getResponseBody();
+        final String clientIdPattern = ",client_id:\"(.*?)\"";
+
+        Document doc = Jsoup.parse(response);
+        final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
+        // The one containing the client id will likely be the last one
+        Collections.reverse(possibleScripts);
+
+        final HashMap<String, String> headers = new HashMap<>();
+        headers.put("Range", "bytes=0-16384");
+
+        for (Element element : possibleScripts) {
+            final String srcUrl = element.attr("src");
+            if (srcUrl != null && !srcUrl.isEmpty()) {
+                try {
+                    return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
+                } catch (RegexException ignored) {
+                    // Ignore it and proceed to try searching other script
+                }
+            }
+        }
+
+        // Officially give up
+        throw new ExtractionException("Couldn't extract client id");
+    }
+
+    static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
+        final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
+        // Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
+        return dl.head(apiUrl).getResponseCode() == 200;
     }
 
     public static String toDateString(String time) throws ParsingException {
@@ -79,7 +101,7 @@ public class SoundcloudParsingHelper {
      * 
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
-    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
+    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
         String apiUrl = "https://api.soundcloud.com/resolve"
                 + "?url=" + URLEncoder.encode(url, "UTF-8")
                 + "&client_id=" + clientId();
@@ -113,7 +135,10 @@ public class SoundcloudParsingHelper {
 
         String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
                 + URLEncoder.encode(url, "UTF-8"));
-        return Parser.matchGroup1(",\"id\":(.*?),", response);
+        // handle playlists / sets different and get playlist id via uir field in JSON
+        if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
+            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
+        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -71,13 +71,15 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
                     final String thumbnailUrl = item.getThumbnailUrl();
                     if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
 
-                    return thumbnailUrl;
+                    String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
+                    return thumbnailUrlBetterResolution;
                 }
             } catch (Exception ignored) {
             }
         }
 
-        return artworkUrl;
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -32,7 +32,10 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
         // Over-engineering at its finest
         if (itemObject.isString(ARTWORK_URL_KEY)) {
             final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
-            if (!artworkUrl.isEmpty()) return artworkUrl;
+            if (!artworkUrl.isEmpty()) {
+                String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                return artworkUrlBetterResolution;
+            }
         }
 
         try {
@@ -42,8 +45,11 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
 
                 // First look for track artwork url
                 if (trackObject.isString(ARTWORK_URL_KEY)) {
-                    final String url = trackObject.getString(ARTWORK_URL_KEY, "");
+                    String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
-                    if (!url.isEmpty()) return url;
+                    if (!artworkUrl.isEmpty()) {
+                        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                        return artworkUrlBetterResolution;
+                    }
                 }
 
                 // Then look for track creator avatar url

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
@@ -48,10 +49,10 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
 
         } catch (UnsupportedEncodingException e) {
             throw new ParsingException("Could not encode query", e);
-        } catch (IOException e) {
-            throw new ParsingException("Could not get client id", e);
         } catch (ReCaptchaException e) {
             throw new ParsingException("ReCaptcha required", e);
+        } catch (IOException | ExtractionException e) {
+            throw new ParsingException("Could not get client id", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -95,6 +95,7 @@ public class SoundcloudService extends StreamingService {
         try {
             list.addKioskEntry(chartsFactory, h, "Top 50");
             list.addKioskEntry(chartsFactory, h, "New & hot");
+            list.setDefaultKiosk("New & hot");
         } catch (Exception e) {
             throw new ExtractionException(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -57,7 +57,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getThumbnailUrl() {
-        return track.getString("artwork_url", "");
+        String artworkUrl = track.getString("artwork_url", "");
+        if (artworkUrl.isEmpty()) {
+            artworkUrl = track.getObject("user").getString("avatar_url", "");
+        }
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -52,7 +52,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
 
     @Override
     public String getThumbnailUrl() {
-        return itemObject.getString("artwork_url");
+        String artworkUrl = itemObject.getString("artwork_url", "");
+        if (artworkUrl.isEmpty()) {
+            artworkUrl = itemObject.getObject("user").getString("avatar_url");
+        }
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
@@ -47,8 +49,9 @@ import java.util.ArrayList;
 
 @SuppressWarnings("WeakerAccess")
 public class YoutubeChannelExtractor extends ChannelExtractor {
+    /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
     private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
-    private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
+    private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
 
     private Document doc;
 
@@ -59,8 +62,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
-        String pageContent = downloader.download(channelUrl);
+        final DownloadResponse response = downloader.get(channelUrl);
-        doc = Jsoup.parse(pageContent, channelUrl);
+        doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
     }
 
     @Override
@@ -72,7 +75,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getUrl() throws ParsingException {
         try {
-            return "https://www.youtube.com/channel/" + getId();
+            return CHANNEL_URL_BASE + getId();
         } catch (ParsingException e) {
             return super.getUrl();
         }
@@ -81,6 +84,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Nonnull
     @Override
     public String getId() throws ParsingException {
+        try {
+            return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
+        } catch (Exception ignored) {}
+
+        // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
         try {
             Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
             if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
@@ -134,10 +142,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     @Override
     public long getSubscriberCount() throws ParsingException {
+
         final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
         if (el != null) {
+            String elTitle = el.attr("title");
             try {
-                return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
+                return Utils.mixedNumberWordToLong(elTitle);
             } catch (NumberFormatException e) {
                 throw new ParsingException("Could not get subscriber count", e);
             }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java

@@ -5,6 +5,9 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 /*
  * Created by Christian Schabesberger on 12.02.17.
  *
@@ -53,8 +56,26 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
 
     @Override
     public String getUrl() throws ParsingException {
-        return el.select("a[class*=\"yt-uix-tile-link\"]").first()
+        try {
-                .attr("abs:href");
+            String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
+                    .attr("abs:data-href");
+
+            Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
+            Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
+
+            if (match.matches()) {
+                return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
+            }
+        } catch(Exception ignored) {}
+
+        // fallback method for channels without "Subscribe" button (or just in case yt changes things)
+        // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
+        try {
+            return el.select("a[class*=\"yt-uix-tile-link\"]").first()
+                    .attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel url", e);
+        }
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -38,7 +38,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         try {
             return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            throw new ParsingException("Could not get author name", e);
+            return "";
         }
     }
 
@@ -95,7 +95,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         try {
             return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            throw new ParsingException("Could not get author name", e);
+            return "";
         }
     }
 
@@ -104,7 +104,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         try {
             return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
         } catch (Exception e) {
-            throw new ParsingException("Could not get author endpoint", e);
+            return "";
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -35,8 +36,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        String pageContent = downloader.download(getUrl());
+        final String url = getUrl();
-        doc = Jsoup.parse(pageContent, getUrl());
+        final DownloadResponse response = downloader.get(url);
+        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
     }
 
     @Override
@@ -50,7 +52,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         try {
             return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
         } catch (Exception e) {
-            throw new ParsingException("Could not get playlist name");
+            throw new ParsingException("Could not get playlist name", e);
         }
     }
 
@@ -59,7 +61,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         try {
             return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
         } catch (Exception e) {
-            throw new ParsingException("Could not get playlist thumbnail");
+            throw new ParsingException("Could not get playlist thumbnail", e);
         }
     }
 
@@ -72,9 +74,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderUrl() throws ParsingException {
         try {
-            return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
+            return YoutubeChannelExtractor.CHANNEL_URL_BASE +
+                    doc.select("button[class*=\"yt-uix-subscription-button\"]")
+                            .first().attr("data-channel-external-id");
         } catch (Exception e) {
-            throw new ParsingException("Could not get playlist uploader name");
+            throw new ParsingException("Could not get playlist uploader url", e);
         }
     }
 
@@ -83,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         try {
             return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
         } catch (Exception e) {
-            throw new ParsingException("Could not get playlist uploader name");
+            throw new ParsingException("Could not get playlist uploader name", e);
         }
     }
 
@@ -92,7 +96,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         try {
             return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
         } catch (Exception e) {
-            throw new ParsingException("Could not get playlist uploader avatar");
+            throw new ParsingException("Could not get playlist uploader avatar", e);
         }
     }
 
@@ -248,6 +252,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
                 @Override
                 public String getUploaderUrl() throws ParsingException {
+                    // this url is not always in the form "/channel/..."
+                    // sometimes Youtube provides urls in the from "/user/..."
                     return getUploaderLink().attr("abs:href");
                 }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java

@@ -49,10 +49,11 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
     @Override
     public String getUrl() throws ParsingException {
         try {
-            final Element div = el.select("div[class=\"yt-lockup-meta\"]").first();
+            final Element a = el.select("div[class=\"yt-lockup-meta\"]")
+                    .select("ul[class=\"yt-lockup-meta-info\"]")
+                    .select("li").select("a").first();
 
-            if(div != null) {
+            if(a != null) {
-                final Element a = div.select("a").first();
                 return a.attr("abs:href");
             }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.utils.Localization;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
@@ -52,13 +54,9 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String site;
         final String url = getUrl();
-        //String url = builder.build().toString();
+        final DownloadResponse response = downloader.get(url, getLocalization());
-        //if we've been passed a valid language code, append it to the URL
+        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
-        site = downloader.download(url, getLocalization());
-
-        doc = Jsoup.parse(site, url);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.*;
 import org.schabi.newpipe.extractor.utils.Localization;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -30,11 +31,13 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /*
  * Created by Christian Schabesberger on 06.08.15.
  *
- * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
  * YoutubeStreamExtractor.java is part of NewPipe.
  *
  * NewPipe is free software: you can redistribute it and/or modify
@@ -64,12 +67,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    public class GemaException extends ContentNotAvailableException {
-        GemaException(String message) {
-            super(message);
-        }
-    }
-
     public class SubtitlesException extends ContentNotAvailableException {
         SubtitlesException(String message, Throwable cause) {
             super(message, cause);
@@ -83,6 +80,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private JsonObject playerArgs;
     @Nonnull
     private final Map<String, String> videoInfoPage = new HashMap<>();
+    private JsonObject playerResponse;
 
     @Nonnull
     private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
@@ -162,14 +160,54 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    // onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;"
+    // :00 is NOT recognized as a timestamp in description or comments.
+    // 0:00 is recognized in both description and comments.
+    // https://www.youtube.com/watch?v=4cccfDXu1vA
+    private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile(
+        "seekTo\\("
+            + "(?:(\\d+)\\*3600\\+)?"  // hours?
+            + "(\\d+)\\*60\\+"  // minutes
+            + "(\\d+)"  // seconds
+            + "\\)");
+
+    @SafeVarargs
+    private static <T> T coalesce(T... args) {
+        for (T arg : args) {
+            if (arg != null) return arg;
+        }
+        throw new IllegalArgumentException("all arguments to coalesce() were null");
+    }
+
     private String parseHtmlAndGetFullLinks(String descriptionHtml)
             throws MalformedURLException, UnsupportedEncodingException, ParsingException {
         final Document description = Jsoup.parse(descriptionHtml, getUrl());
         for(Element a : description.select("a")) {
-            final URL redirectLink = new URL(
+            final String rawUrl = a.attr("abs:href");
-                    a.attr("abs:href"));
+            final URL redirectLink = new URL(rawUrl);
-            final String queryString = redirectLink.getQuery();
+
-            if(queryString != null) {
+            final Matcher onClickTimestamp;
+            final String queryString;
+            if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick")))
+                    .find()) {
+                a.removeAttr("onclick");
+
+                String hours = coalesce(onClickTimestamp.group(1), "0");
+                String minutes = onClickTimestamp.group(2);
+                String seconds = onClickTimestamp.group(3);
+
+                int timestamp = 0;
+                timestamp += Integer.parseInt(hours) * 3600;
+                timestamp += Integer.parseInt(minutes) * 60;
+                timestamp += Integer.parseInt(seconds);
+
+                String setTimestamp = "&t=" + timestamp;
+
+                // Even after clicking https://youtu.be/...?t=6,
+                // getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=.
+                a.attr("href", getUrl() + setTimestamp);
+
+            } else if((queryString = redirectLink.getQuery()) != null) {
                 // if the query string is null we are not dealing with a redirect link,
                 // so we don't need to override it.
                 final String link =
@@ -179,11 +217,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     // if link is null the a tag is a hashtag.
                     // They refer to the youtube search. We do not handle them.
                     a.text(link);
+                    a.attr("href", link);
                 } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
                     a.text(redirectLink.toString());
+                    a.attr("href", redirectLink.toString());
                 }
             } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
+                descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
                 a.text(redirectLink.toString());
+                a.attr("href", redirectLink.toString());
             }
         }
         return description.select("body").first().html();
@@ -206,29 +248,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public long getLength() throws ParsingException {
         assertPageFetched();
-        if(playerArgs != null) {
-            try {
-                long returnValue = Long.parseLong(playerArgs.get("length_seconds") + "");
-                if (returnValue >= 0) return returnValue;
-            } catch (Exception ignored) {
-                // Try other method...
-            }
-        }
 
-        String lengthString = videoInfoPage.get("length_seconds");
+        // try getting duration from playerargs
         try {
-            return Long.parseLong(lengthString);
+            String durationMs = playerResponse
-        } catch (Exception ignored) {
+                    .getObject("streamingData")
-            // Try other method...
+                    .getArray("formats")
-        }
+                    .getObject(0)
-
+                    .getString("approxDurationMs");
-        // TODO: 25.11.17 Implement a way to get the length for age restricted videos #44
+            return Long.parseLong(durationMs)/1000;
-        try {
-            // Fallback to HTML method
-            return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first()
-                    .attr("aria-valuemax"));
         } catch (Exception e) {
-            throw new ParsingException("Could not get video length", e);
+        }
+
+        //try getting value from age gated video
+        try {
+            String duration = playerResponse
+                    .getObject("videoDetails")
+                    .getString("lengthSeconds");
+            return Long.parseLong(duration);
+        } catch (Exception e) {
+            throw new ParsingException("Every methode to get the duration has failed: ", e);
         }
     }
 
@@ -385,31 +424,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getHlsUrl() throws ParsingException {
         assertPageFetched();
-        try {
-            String hlsvp = "";
-            if (playerArgs != null) {
-                if( playerArgs.isString("hlsvp") ) {
-                    hlsvp = playerArgs.getString("hlsvp", "");
-                }else {
-                    hlsvp = JsonParser.object()
-                            .from(playerArgs.getString("player_response", "{}"))
-                            .getObject("streamingData", new JsonObject())
-                            .getString("hlsManifestUrl", "");
-                }
-            }
 
-            return hlsvp;
+        try {
+            return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
         } catch (Exception e) {
-            throw new ParsingException("Could not get hls manifest url", e);
+            if (playerArgs != null && playerArgs.isString("hlsvp")) {
+                return playerArgs.getString("hlsvp");
+            } else {
+                throw new ParsingException("Could not get hls manifest url", e);
+            }
         }
     }
 
     @Override
-    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+    public List<AudioStream> getAudioStreams() throws ExtractionException {
         assertPageFetched();
         List<AudioStream> audioStreams = new ArrayList<>();
         try {
-            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
+            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
                 ItagItem itag = entry.getValue();
 
                 AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
@@ -425,11 +457,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+    public List<VideoStream> getVideoStreams() throws ExtractionException {
         assertPageFetched();
         List<VideoStream> videoStreams = new ArrayList<>();
         try {
-            for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
+            for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
                 ItagItem itag = entry.getValue();
 
                 VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
@@ -449,7 +481,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         assertPageFetched();
         List<VideoStream> videoOnlyStreams = new ArrayList<>();
         try {
-            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
+            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
                 ItagItem itag = entry.getValue();
 
                 VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
@@ -486,7 +518,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         assertPageFetched();
         try {
             if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
-                    playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
+                    (!playerResponse.getObject("streamingData").has(FORMATS)))) {
                 return StreamType.LIVE_STREAM;
             }
         } catch (Exception e) {
@@ -538,21 +570,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
      */
     @Override
     public String getErrorMessage() {
-        String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
         StringBuilder errorReason;
+        Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
 
-        if (errorMessage == null || errorMessage.isEmpty()) {
+        if (errorElement == null) {
             errorReason = null;
-        } else if (errorMessage.contains("GEMA")) {
-            // Gema sometimes blocks youtube music content in germany:
-            // https://www.gema.de/en/
-            // Detailed description:
-            // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
-            errorReason = new StringBuilder("GEMA");
         } else {
-            errorReason = new StringBuilder(errorMessage);
+            String errorMessage = errorElement.text();
-            errorReason.append("  ");
+            if (errorMessage == null || errorMessage.isEmpty()) {
-            errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
+                errorReason = null;
+            } else {
+                errorReason = new StringBuilder(errorMessage);
+                errorReason.append("  ");
+                errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
+            }
         }
 
         return errorReason != null ? errorReason.toString() : null;
@@ -562,8 +593,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     // Fetch page
     //////////////////////////////////////////////////////////////////////////*/
 
-    private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
+    private static final String FORMATS = "formats";
-    private static final String ADAPTIVE_FMTS = "adaptive_fmts";
+    private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
     private static final String HTTPS = "https:";
     private static final String CONTENT = "content";
     private static final String DECRYPTION_FUNC_NAME = "decrypt";
@@ -571,7 +602,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
 
     private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX =
-            "(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
+            "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
     private final static String DECRYPTION_AKAMAIZED_STRING_REGEX =
             "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
     private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX =
@@ -581,22 +612,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private String pageHtml = null;
 
-    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
-        final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
-        if (pageHtml == null) {
-            pageHtml = downloader.download(verifiedUrl);
-        }
-        return pageHtml;
-    }
-
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String pageContent = getPageHtml(downloader);
+        final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
-        doc = Jsoup.parse(pageContent, getUrl());
+        final DownloadResponse response = downloader.get(verifiedUrl);
+        pageHtml = response.getResponseBody();
+        doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
 
         final String playerUrl;
         // Check if the video is age restricted
-        if (pageContent.contains("<meta property=\"og:restrictions:age")) {
+        if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
             final EmbeddedInfo info = getEmbeddedInfo();
             final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
             final String infoPageResponse = downloader.download(videoInfoUrl);
@@ -604,11 +629,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             playerUrl = info.url;
             isAgeRestricted = true;
         } else {
-            final JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
+            final JsonObject ytPlayerConfig = getPlayerConfig();
             playerArgs = getPlayerArgs(ytPlayerConfig);
             playerUrl = getPlayerUrl(ytPlayerConfig);
             isAgeRestricted = false;
         }
+        playerResponse = getPlayerResponse();
 
         if (decryptionCode.isEmpty()) {
             decryptionCode = loadDecryptionCode(playerUrl);
@@ -619,15 +645,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
+    private JsonObject getPlayerConfig() throws ParsingException {
         try {
-            String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
+            String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
             return JsonParser.object().from(ytPlayerConfigRaw);
         } catch (Parser.RegexException e) {
             String errorReason = getErrorMessage();
             switch (errorReason) {
-                case "GEMA":
-                    throw new GemaException(errorReason);
                 case "":
                     throw new ContentNotAvailableException("Content not available: player config empty", e);
                 default:
@@ -670,6 +694,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    private JsonObject getPlayerResponse() throws ParsingException {
+        try {
+            String playerResponseStr;
+            if(playerArgs != null) {
+                playerResponseStr = playerArgs.getString("player_response");
+            } else {
+                playerResponseStr = videoInfoPage.get("player_response");
+            }
+            return JsonParser.object().from(playerResponseStr);
+        } catch (Exception e) {
+            throw new ParsingException("Could not parse yt player response", e);
+        }
+    }
+
     @Nonnull
     private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
         try {
@@ -685,16 +723,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 playerUrl = HTTPS + playerUrl;
             }
 
-            // Get embed sts
+            try {
-            final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
+                // Get embed sts
-            final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+                final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
+                final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+                return new EmbeddedInfo(playerUrl, sts);
+            } catch (Exception i) {
+                // if it failes we simply reply with no sts as then it does not seem to be necessary
+                return new EmbeddedInfo(playerUrl, "");
+            }
 
-            return new EmbeddedInfo(playerUrl, sts);
         } catch (IOException e) {
             throw new ParsingException(
                     "Could load decryption code form restricted video for the Youtube service.", e);
-        } catch (ReCaptchaException e) {
-            throw new ReCaptchaException("reCaptcha Challenge requested");
         }
     }
 
@@ -708,15 +749,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             }
 
             final String playerCode = downloader.download(playerUrl);
+            final String decryptionFunctionName = getDecryptionFuncName(playerCode);
 
-            final String decryptionFunctionName;
-            if (Parser.isMatch(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode)) {
-                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
-            } else if (Parser.isMatch(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode)) {
-                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
-            } else {
-                decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
-            }
             final String functionPattern = "("
                     + decryptionFunctionName.replace("$", "\\$")
                     + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
@@ -757,30 +791,38 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return result == null ? "" : result.toString();
     }
 
+    private String getDecryptionFuncName(String playerCode) throws DecryptException {
+        String decryptionFunctionName;
+        // Cascading things in catch is ugly, but its faster than running a match before getting the actual name
+        // to se if the function can actually be found with the given regex.
+        // However if this cascading should propably be cleaned up somehow as it looks a bit weird.
+        try {
+            decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
+        } catch (Parser.RegexException re) {
+            try {
+                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
+            } catch (Parser.RegexException re2) {
+                try {
+                    decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
+                } catch (Parser.RegexException re3) {
+                    throw new DecryptException("Could not find decrypt function with any of the given patterns.", re);
+                }
+            }
+        }
+        return decryptionFunctionName;
+    }
+
     @Nonnull
     private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
         // If the video is age restricted getPlayerConfig will fail
         if(isAgeRestricted) return Collections.emptyList();
 
-        final JsonObject playerConfig;
-        try {
-            playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
-        } catch (IOException | ExtractionException e) {
-            throw new SubtitlesException("Unable to download player configs", e);
-        }
-        final String playerResponse = playerConfig.getObject("args", new JsonObject())
-                .getString("player_response");
-
         final JsonObject captions;
-        try {
+        if (!playerResponse.has("captions")) {
-            if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
+            // Captions does not exist
-                // Captions does not exist
+            return Collections.emptyList();
-                return Collections.emptyList();
-            }
-            captions = JsonParser.object().from(playerResponse).getObject("captions");
-        } catch (JsonParserException e) {
-            throw new SubtitlesException("Unable to parse subtitles listing", e);
         }
+        captions = playerResponse.getObject("captions");
 
         final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
         final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
@@ -849,39 +891,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 "&sts=" + sts + "&ps=default&gl=US&hl=en";
     }
 
-    private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
+    private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
         Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
-
+        JsonObject streamingData = playerResponse.getObject("streamingData");
-        String encodedUrlMap = "";
+        if (!streamingData.has(streamingDataKey)) {
-        if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
+            return urlAndItags;
-            encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
-        } else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
-            encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
         }
 
-        for (String url_data_str : encodedUrlMap.split(",")) {
+        JsonArray formats = streamingData.getArray(streamingDataKey);
-            try {
+        for (int i = 0; i != formats.size(); ++i) {
-                // This loop iterates through multiple streams, therefore tags
+            JsonObject formatData = formats.getObject(i);
-                // is related to one and the same stream at a time.
+            int itag = formatData.getInt("itag");
-                Map<String, String> tags = Parser.compatParseMap(
-                        org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
 
-                int itag = Integer.parseInt(tags.get("itag"));
+            if (ItagItem.isSupported(itag)) {
-
+                try {
-                if (ItagItem.isSupported(itag)) {
                     ItagItem itagItem = ItagItem.getItag(itag);
                     if (itagItem.itagType == itagTypeWanted) {
-                        String streamUrl = tags.get("url");
+                        String streamUrl;
-                        // if video has a signature: decrypt it and add it to the url
+                        if (formatData.has("url")) {
-                        if (tags.get("s") != null) {
+                            streamUrl = formatData.getString("url");
-                            streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
+                        } else {
+                            // this url has an encrypted signature
+                            Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
+                            streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
                         }
+
                         urlAndItags.put(streamUrl, itagItem);
                     }
+                } catch (UnsupportedEncodingException ignored) {
+
                 }
-            } catch (DecryptException e) {
-                throw e;
-            } catch (Exception ignored) {
             }
         }
 
@@ -954,4 +993,61 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             }
         };
     }
+
+	@Nonnull
+	@Override
+	public List<Frameset> getFrames() throws ExtractionException {
+		try {
+			final String script = doc.select("#player-api").first().siblingElements().select("script").html();
+			int p = script.indexOf("ytplayer.config");
+			if (p == -1) {
+				return Collections.emptyList();
+			}
+			p = script.indexOf('{', p);
+			int e = script.indexOf("ytplayer.load", p);
+			if (e == -1) {
+				return Collections.emptyList();
+			}
+			JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
+			final String resp = jo.getObject("args").getString("player_response");
+			jo = JsonParser.object().from(resp);
+			final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
+			final String url = spec[0];
+			final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
+			for (int i = 1; i < spec.length; ++i) {
+				final String[] parts = spec[i].split("#");
+				if (parts.length != 8) {
+					continue;
+				}
+				final int frameWidth = Integer.parseInt(parts[0]);
+				final int frameHeight = Integer.parseInt(parts[1]);
+				final int totalCount = Integer.parseInt(parts[2]);
+				final int framesPerPageX = Integer.parseInt(parts[3]);
+				final int framesPerPageY = Integer.parseInt(parts[4]);
+				final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
+				final List<String> urls;
+				if (baseUrl.contains("$M")) {
+					final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
+					urls = new ArrayList<>(totalPages);
+					for (int j = 0; j < totalPages; j++) {
+						urls.add(baseUrl.replace("$M", String.valueOf(j)));
+					}
+				} else {
+					urls = Collections.singletonList(baseUrl);
+				}
+				result.add(new Frameset(
+						urls,
+						frameWidth,
+						frameHeight,
+						totalCount,
+						framesPerPageX,
+						framesPerPageY
+				));
+			}
+			result.trimToSize();
+			return result;
+		} catch (Exception e) {
+			throw new ExtractionException(e);
+		}
+	}
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -61,7 +61,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
     @Override
     public String getUrl() throws ParsingException {
         try {
-            Element el = item.select("div[class*=\"yt-lockup-video\"").first();
+            Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
             Element dl = el.select("h3").first().select("a").first();
             return dl.attr("abs:href");
         } catch (Exception e) {
@@ -72,7 +72,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
     @Override
     public String getName() throws ParsingException {
         try {
-            Element el = item.select("div[class*=\"yt-lockup-video\"").first();
+            Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
             Element dl = el.select("h3").first().select("a").first();
             return dl.text();
         } catch (Exception e) {
@@ -107,6 +107,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
 
     @Override
     public String getUploaderUrl() throws ParsingException {
+        // this url is not always in the form "/channel/..."
+        // sometimes Youtube provides urls in the from "/user/..."
         try {
             try {
                 return item.select("div[class=\"yt-lockup-byline\"]").first()
@@ -119,7 +121,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
                     .text().split(" - ")[0];
         } catch (Exception e) {
             System.out.println(item.html());
-            throw new ParsingException("Could not get uploader", e);
+            throw new ParsingException("Could not get uploader url", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java

@@ -63,16 +63,10 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
             String title = outline.attr("title");
             String xmlUrl = outline.attr("abs:xmlUrl");
 
-            if (title.isEmpty() || xmlUrl.isEmpty()) {
-                throw new InvalidSourceException("document has invalid entries");
-            }
-
             try {
                 String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
                 result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
-            } catch (Parser.RegexException e) {
+            } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
-                throw new InvalidSourceException("document has invalid entries", e);
-            }
         }
 
         return result;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -24,12 +24,14 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Localization;
@@ -56,8 +58,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
             url += "?gl=" + contentCountry;
         }
 
-        String pageContent = downloader.download(url);
+        final DownloadResponse response = downloader.get(url);
-        doc = Jsoup.parse(pageContent, url);
+        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
     }
 
     @Override
@@ -126,6 +128,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
                     }
 
                     private Element getUploaderLink() {
+                        // this url is not always in the form "/channel/..."
+                        // sometimes Youtube provides urls in the from "/user/..."
                         Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
                         return uploaderEl.select("a").first();
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java

@@ -46,7 +46,8 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
             URL urlObj = Utils.stringToURL(url);
             String path = urlObj.getPath();
 
-            if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) {
+            if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
+                    YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) {
                 throw new ParsingException("the URL given is not a Youtube-URL");
             }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -1,7 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
 
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 
 import java.net.URL;
 
@@ -30,40 +34,42 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
-    private static boolean isHTTP(URL url) {
+    private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
-        // make sure its http or https
+            "form[action*=\"/das_captcha\"]",
-        String protocol = url.getProtocol();
+            "input[name*=\"action_recaptcha_verify\"]"
-        if (!protocol.equals("http") && !protocol.equals("https")) {
+    };
-            return false;
+
+    public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException {
+        final Document document = Jsoup.parse(response.getResponseBody(), url);
+
+        for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
+            if (!document.select(detectionSelector).isEmpty()) {
+                throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url);
+            }
         }
 
-        boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
+        return document;
-        boolean setsNoPort = url.getPort() == -1;
-
-        return setsNoPort || usesDefaultPort;
     }
 
     public static boolean isYoutubeURL(URL url) {
-        // make sure its http or https
-        if (!isHTTP(url))
-            return false;
-
-        // make sure its a known youtube url
         String host = url.getHost();
         return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
-                || host.equalsIgnoreCase("m.youtube.com");
+                || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
     }
 
-    public static boolean isYoutubeALikeURL(URL url) {
+    public static boolean isYoutubeServiceURL(URL url) {
-        // make sure its http or https
-        if (!isHTTP(url))
-            return false;
-
-        // make sure its a known youtube url
         String host = url.getHost();
-        return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
+        return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
-                || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("www.youtube-nocookie.com")
+    }
-                || host.equalsIgnoreCase("youtu.be") || host.equalsIgnoreCase("hooktube.com");
+
+    public static boolean isHooktubeURL(URL url) {
+        String host = url.getHost();
+        return host.equalsIgnoreCase("hooktube.com");
+    }
+
+    public static boolean isInvidioURL(URL url) {
+        String host = url.getHost();
+        return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
     }
 
     public static long parseDurationString(String input)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java

@@ -25,10 +25,16 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
         try {
             URL urlObj = Utils.stringToURL(url);
 
-            if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) {
+            if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
+                    || YoutubeParsingHelper.isInvidioURL(urlObj))) {
                 throw new ParsingException("the url given is not a Youtube-URL");
             }
 
+            String path = urlObj.getPath();
+            if (!path.equals("/watch" ) && !path.equals("/playlist")) {
+                throw new ParsingException("the url given is neither a video nor a playlist URL");
+            }
+
             String listID = Utils.getQueryValue(urlObj, "list");
 
             if (listID == null) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -60,7 +60,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             URI uri = new URI(urlString);
             String scheme = uri.getScheme();
 
-            if (scheme != null && scheme.equals("vnd.youtube")) {
+            if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
                 String schemeSpecificPart = uri.getSchemeSpecificPart();
                 if (schemeSpecificPart.startsWith("//")) {
                     urlString = "https:" + schemeSpecificPart;
@@ -85,7 +85,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             path = path.substring(1);
         }
 
-        if (!YoutubeParsingHelper.isYoutubeALikeURL(url)) {
+        if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
+                YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
+                YoutubeParsingHelper.isInvidioURL(url))) {
             if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
                 throw new FoundAdException("Error found ad: " + urlString);
             }
@@ -112,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
             case "YOUTUBE.COM":
             case "WWW.YOUTUBE.COM":
-            case "M.YOUTUBE.COM": {
+            case "M.YOUTUBE.COM":
+            case "MUSIC.YOUTUBE.COM": {
                 if (path.equals("attribution_link")) {
                     String uQueryValue = Utils.getQueryValue(url, "u");
 
@@ -147,6 +150,34 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             }
 
             case "HOOKTUBE.COM": {
+                if (path.startsWith("v/")) {
+                    String id = path.substring("v/".length());
+
+                    return assertIsID(id);
+                }
+                if (path.startsWith("watch/")) {
+                    String id = path.substring("watch/".length());
+
+                    return assertIsID(id);
+                }
+                // there is no break-statement here on purpose so the next code-block gets also run for hooktube
+            }
+
+            case "WWW.INVIDIO.US":
+            case "DEV.INVIDIO.US":
+            case "INVIDIO.US":
+            case "INVIDIOUS.SNOPYTA.ORG":
+            case "DE.INVIDIOUS.SNOPYTA.ORG":
+            case "FI.INVIDIOUS.SNOPYTA.ORG":
+            case "VID.WXZM.SX":
+            case "INVIDIOUS.KABI.TK":
+            case "INVIDIOU.SH":
+            case "WWW.INVIDIOU.SH":
+            case "NO.INVIDIOU.SH":
+            case "INVIDIOUS.ENKIRTON.NET":
+            case "TUBE.POAL.CO":
+            case "INVIDIOUS.13AD.DE":
+            case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
                 if (path.equals("watch")) {
                     String viewQueryValue = Utils.getQueryValue(url, "v");
                     if (viewQueryValue != null) {
@@ -158,19 +189,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
                     return assertIsID(id);
                 }
-                if (path.startsWith("v/")) {
-                    String id = path.substring("v/".length());
 
-                    return assertIsID(id);
+                break;
-                }
-                if (path.startsWith("watch/")) {
-                    String id = path.substring("watch/".length());
-
-                    return assertIsID(id);
-                }
             }
-
-            break;
         }
 
         throw new ParsingException("Error no suitable url: " + urlString);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java

@@ -48,6 +48,6 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
         }
 
         String urlPath = urlObj.getPath();
-        return YoutubeParsingHelper.isYoutubeURL(urlObj) && urlPath.equals("/feed/trending");
+        return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending");
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java

@@ -0,0 +1,66 @@
+package org.schabi.newpipe.extractor.stream;
+
+import javax.annotation.Nullable;
+import java.util.Collection;
+import java.util.List;
+
+public final class Frameset {
+
+	private List<String> urls;
+	private int frameWidth;
+	private int frameHeight;
+	private int totalCount;
+	private int framesPerPageX;
+	private int framesPerPageY;
+
+	public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
+		this.urls = urls;
+		this.totalCount = totalCount;
+		this.frameWidth = frameWidth;
+		this.frameHeight = frameHeight;
+		this.framesPerPageX = framesPerPageX;
+		this.framesPerPageY = framesPerPageY;
+	}
+
+	/**
+	 * @return list of urls to images with frames
+	 */
+	public List<String> getUrls() {
+		return urls;
+	}
+
+	/**
+	 * @return total count of frames
+	 */
+	public int getTotalCount() {
+		return totalCount;
+	}
+
+	/**
+	 * @return maximum frames count by x
+	 */
+	public int getFramesPerPageX() {
+		return framesPerPageX;
+	}
+
+	/**
+	 * @return maximum frames count by y
+	 */
+	public int getFramesPerPageY() {
+		return framesPerPageY;
+	}
+
+	/**
+	 * @return width of a one frame, in pixels
+	 */
+	public int getFrameWidth() {
+		return frameWidth;
+	}
+
+	/**
+	 * @return height of a one frame, in pixels
+	 */
+	public int getFrameHeight() {
+		return frameHeight;
+	}
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -30,7 +30,10 @@ import org.schabi.newpipe.extractor.utils.Localization;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -256,7 +259,18 @@ public abstract class StreamExtractor extends Extractor {
     public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
 
     /**
-     * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
+     * Should return a list of Frameset object that contains preview of stream frames
+     * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream
+     * @throws IOException
+     * @throws ExtractionException
+     */
+    @Nonnull
+    public List<Frameset> getFrames() throws IOException, ExtractionException {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Should analyse the webpage's document and extracts any error message there might be.
      *
      * @return Error message; null if there is no error message.
      */

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -123,8 +123,6 @@ public class DashMpdParser {
             dashDoc = downloader.download(streamInfo.getDashMpdUrl());
         } catch (IOException ioe) {
             throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
-        } catch (ReCaptchaException e) {
-            throw new ReCaptchaException("reCaptcha Challenge needed");
         }
 
         try {

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -27,6 +27,38 @@ public class Utils {
         return toRemove.replaceAll("\\D+", "");
     }
 
+    /**
+     * <p>Convert a mixed number word to a long.</p>
+     * <p>Examples:</p>
+     * <ul>
+     *     <li>123 -&gt; 123</li>
+     *     <li>1.23K -&gt; 1230</li>
+     *     <li>1.23M -&gt; 1230000</li>
+     * </ul>
+     * @param numberWord string to be converted to a long
+     * @return a long
+     * @throws NumberFormatException
+     * @throws ParsingException
+     */
+    public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
+        String multiplier = "";
+        try {
+            multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
+        } catch(ParsingException ignored) {}
+        double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)
+                .replace(",", "."));
+        switch (multiplier.toUpperCase()) {
+            case "K":
+                return (long) (count * 1e3);
+            case "M":
+                return (long) (count * 1e6);
+            case "B":
+                return (long) (count * 1e9);
+            default:
+                return (long) (count);
+        }
+    }
+
     /**
      * Check if the url matches the pattern.
      *
@@ -121,6 +153,19 @@ public class Utils {
         }
     }
 
+    public static boolean isHTTP(URL url) {
+        // make sure its http or https
+        String protocol = url.getProtocol();
+        if (!protocol.equals("http") && !protocol.equals("https")) {
+            return false;
+        }
+
+        boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
+        boolean setsNoPort = url.getPort() == -1;
+
+        return setsNoPort || usesDefaultPort;
+    }
+    
     public static String removeUTF8BOM(String s) {
         if (s.startsWith("\uFEFF")) {
             s = s.substring(1);

extractor/src/test/java/org/schabi/newpipe/Downloader.java

@@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.utils.Localization;
 
+import static java.util.Collections.singletonList;
+
 /*
  * Created by Christian Schabesberger on 28.01.16.
  *
@@ -129,7 +131,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
              * request See : https://github.com/rg3/youtube-dl/issues/5138
              */
             if (con.getResponseCode() == 429) {
-                throw new ReCaptchaException("reCaptcha Challenge requested");
+                throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
             }
 
             throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
@@ -172,6 +174,36 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         return dl(con);
     }
 
+    @Override
+    public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException {
+        final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection();
+
+        try {
+            con.setRequestMethod("HEAD");
+            setDefaults(con);
+        } catch (Exception e) {
+            /*
+             * HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
+             * request See : https://github.com/rg3/youtube-dl/issues/5138
+             */
+            if (con.getResponseCode() == 429) {
+                throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
+            }
+
+            throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
+        }
+
+        return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields());
+    }
+
+    @Override
+    public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
+        final Map<String, List<String>> requestHeaders = new HashMap<>();
+        requestHeaders.put("Accept-Language", singletonList(localization.getLanguage()));
+
+        return get(siteUrl, new DownloadRequest(null, requestHeaders));
+    }
+
     @Override
     public DownloadResponse get(String siteUrl, DownloadRequest request)
             throws IOException, ReCaptchaException {
@@ -183,7 +215,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
             }
         }
         String responseBody = dl(con);
-        return new DownloadResponse(responseBody, con.getHeaderFields());
+        return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields());
     }
 
     @Override
@@ -219,6 +251,6 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
                 sb.append(inputLine);
             }
         }
-        return new DownloadResponse(sb.toString(), con.getHeaderFields());
+        return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java

@@ -36,7 +36,7 @@ public class MediaCCCOggTest {
     @Test
     public void getAudioStreamsContainOgg() throws Exception {
         for(AudioStream stream : extractor.getAudioStreams()) {
-            System.out.println(stream.getFormat());
+            assertEquals("OGG", stream.getFormat().toString());
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java

@@ -25,7 +25,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
  * Test for {@link PeertubeChannelExtractor}
  */
 public class PeertubeChannelExtractorTest {
-    public static class LilUzi implements BaseChannelExtractorTest {
+    public static class KDE implements BaseChannelExtractorTest {
         private static PeertubeChannelExtractor extractor;
 
         @BeforeClass
@@ -34,7 +34,7 @@ public class PeertubeChannelExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
             extractor = (PeertubeChannelExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org");
+                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
             extractor.fetchPage();
         }
 
@@ -49,22 +49,22 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testName() throws ParsingException {
-            assertEquals("Noureddine HADDAG", extractor.getName());
+            assertEquals("The KDE Community", extractor.getName());
         }
 
         @Test
         public void testId() throws ParsingException {
-            assertEquals("root@tube.openalgeria.org", extractor.getId());
+            assertEquals("kde", extractor.getId());
         }
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
         }
 
         @Test
         public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl());
+            assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -112,7 +112,7 @@ public class PeertubeChannelExtractorTest {
         }
     }
 
-    public static class DubMatix implements BaseChannelExtractorTest {
+    public static class Booteille implements BaseChannelExtractorTest {
         private static PeertubeChannelExtractor extractor;
 
         @BeforeClass
@@ -121,7 +121,7 @@ public class PeertubeChannelExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
             extractor = (PeertubeChannelExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch");
+                    .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
             extractor.fetchPage();
         }
 
@@ -146,22 +146,22 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testName() throws ParsingException {
-            assertEquals("France Inter", extractor.getName());
+            assertEquals("booteille", extractor.getName());
         }
 
         @Test
         public void testId() throws ParsingException {
-            assertEquals("franceinter@tube.kdy.ch", extractor.getId());
+            assertEquals("booteille", extractor.getId());
         }
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
         }
 
         @Test
         public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl());
+            assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -205,7 +205,7 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testSubscriberCount() throws ParsingException {
-            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75);
+            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java

@@ -32,7 +32,7 @@ public class PeertubeStreamExtractorDefaultTest {
         NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
         // setting instance might break test when running in parallel
         PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
-        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
+        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
         extractor.fetchPage();
     }
 
@@ -44,39 +44,39 @@ public class PeertubeStreamExtractorDefaultTest {
 
     @Test
     public void testGetTitle() throws ParsingException {
-        assertEquals(extractor.getName(), "The Internet's Own Boy");
+        assertEquals(extractor.getName(), "Power Corrupts the Best");
     }
 
     @Test
     public void testGetDescription() throws ParsingException {
-        assertEquals(extractor.getDescription(), "The story of programming prodigy and information activist Aaron Swartz, who took his own life at the age of 26.");
+        assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
     }
 
     @Test
     public void testGetUploaderName() throws ParsingException {
-        assertEquals(extractor.getUploaderName(), "root");
+        assertEquals(extractor.getUploaderName(), "Rowsedower");
     }
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(extractor.getLength(), 6299);
+        assertEquals(extractor.getLength(), 269);
     }
 
     @Test
     public void testGetViewCount() throws ParsingException {
         assertTrue(Long.toString(extractor.getViewCount()),
-                extractor.getViewCount() > 700);
+                extractor.getViewCount() > 10);
     }
 
     @Test
     public void testGetUploadDate() throws ParsingException {
-        assertEquals("2017-10-17", extractor.getUploadDate());
+        assertEquals("2018-09-30", extractor.getUploadDate());
     }
 
     @Test
     public void testGetUploaderUrl() throws ParsingException {
         assertIsSecureUrl(extractor.getUploaderUrl());
-        assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@peertube2.cpy.re", extractor.getUploaderUrl());
+        assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java

@@ -25,23 +25,21 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
         NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
         // setting instance might break test when running in parallel
         PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
-        extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy");
+        extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde");
         extractor.fetchPage();
         itemsPage = extractor.getInitialPage();
     }
 
     @Test
     public void testGetSecondPageUrl() throws Exception {
-        assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet%27s+own+boy&start=12&count=12", extractor.getNextPageUrl());
+        assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl());
     }
 
     @Test
     public void testResultList_FirstElement() {
         InfoItem firstInfoItem = itemsPage.getItems().get(0);
         
-        // THe channel should be the first item
+        assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde"));
-        assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName());
-        assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl());
     }
 
     @Test
@@ -83,11 +81,11 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
 
     @Test
     public void testId() throws Exception {
-        assertEquals("internet's own boy", extractor.getId());
+        assertEquals("kde", extractor.getId());
     }
 
     @Test
     public void testName() {
-        assertEquals("internet's own boy", extractor.getName());
+        assertEquals("kde", extractor.getName());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -36,7 +36,6 @@ public class SoundcloudChartsExtractorTest {
         assertNotNull(NewPipe.getDownloader());
     }
 
-    @Ignore
     @Test
     public void testGetName() throws Exception {
         assertEquals(extractor.getName(), "Top 50");

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -1,18 +1,24 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.junit.Assert;
+import org.junit.*;
-import org.junit.BeforeClass;
-import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.utils.Localization;
 
+import static org.junit.Assert.*;
+
 public class SoundcloudParsingHelperTest {
     @BeforeClass
     public static void setUp() {
         NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
     }
 
+    @Test
+    public void assertThatHardcodedClientIdIsValid() throws Exception {
+        assertTrue("Hardcoded client id is not valid anymore",
+                SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(Downloader.getInstance()));
+    }
+
     @Test
     public void resolveUrlWithEmbedPlayerTest() throws Exception {
         Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import org.hamcrest.CoreMatchers;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractorTest {
         }
     }
 
-    public static class RandomHouseDanceMusic implements BasePlaylistExtractorTest {
+    public static class RandomHouseMusic implements BasePlaylistExtractorTest {
         private static SoundcloudPlaylistExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
             extractor = (SoundcloudPlaylistExtractor) SoundCloud
-                    .getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2");
+                    .getPlaylistExtractor("https://soundcloud.com/micky96/sets/house");
             extractor.fetchPage();
         }
 
@@ -141,22 +142,22 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testName() {
-            assertEquals("House, Electro , Dance Music 2", extractor.getName());
+            assertEquals("House", extractor.getName());
         }
 
         @Test
         public void testId() {
-            assertEquals("310980722", extractor.getId());
+            assertEquals("123062856", extractor.getId());
         }
 
         @Test
         public void testUrl() throws Exception {
-            assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl());
+            assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getUrl());
         }
 
         @Test
         public void testOriginalUrl() throws Exception {
-            assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl());
+            assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -182,7 +183,7 @@ public class SoundcloudPlaylistExtractorTest {
             assertIsSecureUrl(extractor.getThumbnailUrl());
         }
 
-        @Ignore
+        @Ignore("not implemented")
         @Test
         public void testBannerUrl() {
             assertIsSecureUrl(extractor.getBannerUrl());
@@ -192,12 +193,12 @@ public class SoundcloudPlaylistExtractorTest {
         public void testUploaderUrl() {
             final String uploaderUrl = extractor.getUploaderUrl();
             assertIsSecureUrl(uploaderUrl);
-            assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader"));
+            assertThat(uploaderUrl, CoreMatchers.containsString("micky96"));
         }
 
         @Test
         public void testUploaderName() {
-            assertEquals("Gosu", extractor.getUploaderName());
+            assertEquals("_mickyyy", extractor.getUploaderName());
         }
 
         @Test
@@ -226,6 +227,7 @@ public class SoundcloudPlaylistExtractorTest {
         // Additional Testing
         //////////////////////////////////////////////////////////////////////////*/
 
+        @Ignore
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
@@ -265,11 +267,14 @@ public class SoundcloudPlaylistExtractorTest {
         // ListExtractor
         //////////////////////////////////////////////////////////////////////////*/
 
+        @Ignore
         @Test
         public void testRelatedItems() throws Exception {
             defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
         }
 
+        //TODO: FUCK THIS: This triggers a 500 at sever
+        @Ignore
         @Test
         public void testMoreRelatedItems() throws Exception {
             ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());
@@ -284,6 +289,7 @@ public class SoundcloudPlaylistExtractorTest {
         // PlaylistExtractor
         //////////////////////////////////////////////////////////////////////////*/
 
+        @Ignore
         @Test
         public void testThumbnailUrl() {
             assertIsSecureUrl(extractor.getThumbnailUrl());

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java

@@ -39,27 +39,27 @@ public class SoundcloudStreamExtractorDefaultTest {
     @Test
     public void testGetValidTimeStamp() throws IOException, ExtractionException {
         StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69");
-        assertEquals(extractor.getTimeStamp() + "", "69");
+        assertEquals("69", extractor.getTimeStamp() + "");
     }
 
     @Test
     public void testGetTitle() throws ParsingException {
-        assertEquals(extractor.getName(), "Do What I Want [Produced By Maaly Raw + Don Cannon]");
+        assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName());
     }
 
     @Test
     public void testGetDescription() throws ParsingException {
-        assertEquals(extractor.getDescription(), "The Perfect LUV Tape®️");
+        assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
     }
 
     @Test
     public void testGetUploaderName() throws ParsingException {
-        assertEquals(extractor.getUploaderName(), "LIL UZI VERT");
+        assertEquals("LIL UZI VERT", extractor.getUploaderName());
     }
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(extractor.getLength(), 175);
+        assertEquals(175, extractor.getLength());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -109,6 +109,7 @@ public class YoutubeChannelExtractorTest {
         @Test
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
+            assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6);
         }
     }
 
@@ -199,6 +200,7 @@ public class YoutubeChannelExtractorTest {
         @Test
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
+            assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6);
         }
 
     }
@@ -394,6 +396,100 @@ public class YoutubeChannelExtractorTest {
         }
     }
 
+    // this channel has no "Subscribe" button
+    public static class EminemVEVO implements BaseChannelExtractorTest {
+        private static YoutubeChannelExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
+            extractor = (YoutubeChannelExtractor) YouTube
+                    .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() throws Exception {
+            assertEquals("EminemVEVO", extractor.getName());
+        }
+
+        @Test
+        public void testId() throws Exception {
+            assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
+        }
+
+        @Test
+        public void testUrl() throws ParsingException {
+            assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws ParsingException {
+            assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+        }
+
+        @Test
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor, YouTube.getServiceId());
+        }
+
+         /*//////////////////////////////////////////////////////////////////////////
+         // ChannelExtractor
+         //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testDescription() throws Exception {
+            final String description = extractor.getDescription();
+            assertTrue(description, description.contains("Eminem on Vevo"));
+        }
+
+        @Test
+        public void testAvatarUrl() throws Exception {
+            String avatarUrl = extractor.getAvatarUrl();
+            assertIsSecureUrl(avatarUrl);
+            assertTrue(avatarUrl, avatarUrl.contains("yt3"));
+        }
+
+        @Test
+        public void testBannerUrl() throws Exception {
+            String bannerUrl = extractor.getBannerUrl();
+            assertIsSecureUrl(bannerUrl);
+            assertTrue(bannerUrl, bannerUrl.contains("yt3"));
+        }
+
+        @Test
+        public void testFeedUrl() throws Exception {
+            assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
+        }
+
+        @Test
+        public void testSubscriberCount() throws Exception {
+            // there is no "Subscribe" button
+            long subscribers = extractor.getSubscriberCount();
+            assertEquals("Wrong subscriber count", -1, subscribers);
+        }
+    }
+
+
+
     public static class RandomChannel implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 
@@ -485,8 +581,9 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testSubscriberCount() throws Exception {
-            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
+            long subscribers = extractor.getSubscriberCount();
+            assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
         }
     }
-};
+}
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java

@@ -37,6 +37,12 @@ public class YoutubeChannelLinkHandlerFactoryTest {
 
         assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA"));
         assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
+    
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh"));
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos"));
+    
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA"));
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
     }
 
     @Test
@@ -53,5 +59,11 @@ public class YoutubeChannelLinkHandlerFactoryTest {
 
         assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
         assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
+    
+        assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId());
+        assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId());
+    
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId());
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -27,18 +27,18 @@ public class YoutubeCommentsExtractorTest {
     public static void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
         extractor = (YoutubeCommentsExtractor) YouTube
-                .getCommentsExtractor("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
+                .getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
     }
 
     @Test
     public void testGetComments() throws IOException, ExtractionException {
-        boolean result = false;
+        boolean result;
         InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
-        result = findInComments(comments, "Tsuki blyat");
+        result = findInComments(comments, "s1ck m3m3");
 
         while (comments.hasNextPage() && !result) {
             comments = extractor.getPage(comments.getNextPageUrl());
-            result = findInComments(comments, "Tsuki blyat");
+            result = findInComments(comments, "s1ck m3m3");
         }
 
         assertTrue(result);
@@ -47,14 +47,14 @@ public class YoutubeCommentsExtractorTest {
     @Test
     public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
         boolean result = false;
-        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
+        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o");
-        assertTrue("the dark side of YouTube...".equals(commentsInfo.getName()));
+        assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
-        result = findInComments(commentsInfo.getRelatedItems(), "Tsuki blyat");
+        result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
 
         String nextPage = commentsInfo.getNextPageUrl();
         while (!StringUtil.isBlank(nextPage) && !result) {
             InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
-            result = findInComments(moreItems.getItems(), "Tsuki blyat");
+            result = findInComments(moreItems.getItems(), "s1ck m3m3");
             nextPage = moreItems.getNextPageUrl();
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -100,7 +100,7 @@ public class YoutubePlaylistExtractorTest {
 
         @Test
         public void testUploaderUrl() throws Exception {
-            assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
+            assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl());
         }
 
         @Test
@@ -185,8 +185,8 @@ public class YoutubePlaylistExtractorTest {
         public void testMoreRelatedItems() throws Exception {
             ListExtractor.InfoItemsPage<StreamInfoItem> currentPage
                     = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
-            // Test for 2 more levels
 
+            // test for 2 more levels
             for (int i = 0; i < 2; i++) {
                 currentPage = extractor.getPage(currentPage.getNextPageUrl());
                 defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors());
@@ -214,7 +214,7 @@ public class YoutubePlaylistExtractorTest {
 
         @Test
         public void testUploaderUrl() throws Exception {
-            assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
+            assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl());
         }
 
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java

@@ -0,0 +1,107 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Localization;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test for {@link YoutubePlaylistLinkHandlerFactory}
+ */
+public class YoutubePlaylistLinkHandlerFactoryTest {
+    private static YoutubePlaylistLinkHandlerFactory linkHandler;
+
+    @BeforeClass
+    public static void setUp() {
+        linkHandler = YoutubePlaylistLinkHandlerFactory.getInstance();
+        NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void getIdWithNullAsUrl() throws ParsingException {
+        linkHandler.fromId(null);
+    }
+
+    @Test
+    public void getIdfromYt() throws Exception {
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
+        assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId());
+    }
+
+    @Test
+    public void testAcceptYtUrl() throws ParsingException {
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
+        assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
+    }
+
+    @Test
+    public void testDeniesInvalidYtUrl() throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("www.youtube.com:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("youtube  .    com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+    }
+
+    @Test
+    public void testAcceptInvidioUrl() throws ParsingException {
+        assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
+        assertTrue(linkHandler.acceptUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
+        assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+    }
+
+    @Test
+    public void testDeniesInvalidInvidioUrl() throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("invidio.us:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("invidio  .    us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+    }
+
+    @Test
+    public void testGetInvidioIdfromUrl() throws ParsingException {
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
+        assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java

@@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
+        assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
     }
 
     @Test
@@ -98,13 +99,14 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
         assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
         assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
-
+        assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
-        assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
+        assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
     }
 
     @Test
     public void testAcceptHookUrl() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU"));
+        assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU"));
         assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE"));
         assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
         assertTrue(linkHandler.acceptUrl("hooktube.com/watch/3msbfr6pBNE"));
@@ -115,10 +117,31 @@ public class YoutubeStreamLinkHandlerFactoryTest {
     @Test
     public void testGetHookIdfromUrl() throws ParsingException {
         assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId());
+        assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId());
         assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId());
         assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
         assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId());
         assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId());
         assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId());
     }
+    
+    @Test
+    public void testAcceptInvidioUrl() throws ParsingException {
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU"));
+        assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU"));
+        assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU"));
+        assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE"));
+        assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
+        assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE"));
+    }
+    
+    @Test
+    public void testGetInvidioIdfromUrl() throws ParsingException {
+        assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId());
+        assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId());
+        assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId());
+        assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId());
+        assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
+        assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId());
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java

@@ -59,15 +59,38 @@ public class YoutubeSubscriptionExtractorTest {
         assertTrue(items.isEmpty());
     }
 
+    @Test
+    public void testSubscriptionWithEmptyTitleInSource() throws Exception {
+        String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
+        String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
+                "<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
+                "</outline></body></opml>";
+
+        List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
+        assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
+        assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
+        assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
+    }
+
+    @Test
+    public void testSubscriptionWithInvalidUrlInSource() throws Exception {
+        String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
+                "<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
+                "<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
+                "<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
+                "<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
+                "</outline></body></opml>";
+
+        List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
+        assertTrue(items.isEmpty());
+    }
+
     @Test
     public void testInvalidSourceException() {
         List<String> invalidList = Arrays.asList(
                 "<xml><notvalid></notvalid></xml>",
                 "<opml><notvalid></notvalid></opml>",
                 "<opml><body></body></opml>",
-                "<opml><body><outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/></outline></body></opml>",
-                "<opml><body><outline><outline text=\"invalid\" title=\"url\" type=\"rss\"" +
-                        " xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/></outline></body></opml>",
                 "",
                 null,
                 "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
@@ -78,11 +101,11 @@ public class YoutubeSubscriptionExtractorTest {
                 if (invalidContent != null) {
                     byte[] bytes = invalidContent.getBytes("UTF-8");
                     subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
+                    fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
                 } else {
                     subscriptionExtractor.fromInputStream(null);
+                    fail("Extracting from null String didn't throw an exception");
                 }
-
-                fail("didn't throw exception");
             } catch (Exception e) {
                 // System.out.println(" -> " + e);
                 boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java

@@ -69,6 +69,10 @@ public class YoutubeTrendingLinkHandlerFactoryTest {
         assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending"));
         assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending"));
 
+        assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending"));
+        assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending"));
+        assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending"));
+
         assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending"));
         assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia"));
         assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending"));

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java

@@ -1,6 +1,8 @@
 package org.schabi.newpipe.extractor.services.youtube.search;
 
+import org.hamcrest.CoreMatchers;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.InfoItem;
@@ -53,6 +55,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
         assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl());
     }
 
+    @Ignore
     @Test
     public void testOnlyContainChannels() {
         for(InfoItem item : itemsPage.getItems()) {
@@ -61,4 +64,19 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
             }
         }
     }
+
+    @Test
+    public void testChannelUrl() {
+        for(InfoItem item : itemsPage.getItems()) {
+            if (item instanceof ChannelInfoItem) {
+                ChannelInfoItem channel = (ChannelInfoItem) item;
+
+                if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie
+                    assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl());
+                } else {
+                    assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
+                }
+            }
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java

@@ -73,7 +73,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
         assertTrue((firstInfoItem instanceof ChannelInfoItem)
                 || (secondInfoItem instanceof ChannelInfoItem));
         assertEquals("name", "PewDiePie", channelItem.getName());
-        assertEquals("url","https://www.youtube.com/user/PewDiePie", channelItem.getUrl());
+        assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -71,10 +71,9 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
         assertFalse(extractor.getUploaderName().isEmpty());
     }
 
-    @Ignore // Currently there is no way get the length from restricted videos
     @Test
     public void testGetLength() throws ParsingException {
-        assertTrue(extractor.getLength() > 0);
+        assertEquals(1789, extractor.getLength());
     }
 
     @Test
@@ -97,8 +96,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
         assertIsSecureUrl(extractor.getUploaderAvatarUrl());
     }
 
-    // FIXME: 25.11.17 Are there no streams or are they not listed?
-    @Ignore
     @Test
     public void testGetAudioStreams() throws IOException, ExtractionException {
         // audio streams are not always necessary

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -71,10 +71,9 @@ public class YoutubeStreamExtractorControversialTest {
         assertFalse(extractor.getUploaderName().isEmpty());
     }
 
-    @Ignore // Currently there is no way get the length from restricted videos
     @Test
     public void testGetLength() throws ParsingException {
-        assertTrue(extractor.getLength() > 0);
+        assertEquals(219, extractor.getLength());
     }
 
     @Test
@@ -97,8 +96,6 @@ public class YoutubeStreamExtractorControversialTest {
         assertIsSecureUrl(extractor.getUploaderAvatarUrl());
     }
 
-    // FIXME: 25.11.17 Are there no streams or are they not listed?
-    @Ignore
     @Test
     public void testGetAudioStreams() throws IOException, ExtractionException {
         // audio streams are not always necessary
@@ -113,17 +110,15 @@ public class YoutubeStreamExtractorControversialTest {
         assertTrue(streams.size() > 0);
     }
 
-    @Ignore
     @Test
     public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
         // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-        assertTrue(!extractor.getSubtitlesDefault().isEmpty());
+        assertFalse(extractor.getSubtitlesDefault().isEmpty());
     }
 
-    @Ignore
     @Test
     public void testGetSubtitlesList() throws IOException, ExtractionException {
         // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-        assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty());
+        assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -1,8 +1,9 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.ExtractorAsserts;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.util.List;
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@@ -53,7 +55,7 @@ public class YoutubeStreamExtractorDefaultTest {
         public static void setUp() throws Exception {
             NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
             extractor = (YoutubeStreamExtractor) YouTube
-                    .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
+                    .getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
             extractor.fetchPage();
         }
 
@@ -81,8 +83,8 @@ public class YoutubeStreamExtractorDefaultTest {
         }
 
         @Test
-        public void testGetFullLinksInDescriptlion() throws ParsingException {
+        public void testGetFullLinksInDescription() throws ParsingException {
-            assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt"));
+            assertTrue(extractor.getDescription().contains("http://adele.com"));
             assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi..."));
         }
 
@@ -95,7 +97,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetLength() throws ParsingException {
-            assertTrue(extractor.getLength() > 0);
+            assertEquals(366, extractor.getLength());
         }
 
         @Test
@@ -111,7 +113,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetUploaderUrl() throws ParsingException {
-            assertTrue(extractor.getUploaderUrl().length() > 0);
+            assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
         }
 
         @Test
@@ -231,4 +233,29 @@ public class YoutubeStreamExtractorDefaultTest {
             assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
         }
     }
+
+    public static class FramesTest {
+        private static YoutubeStreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
+            extractor = (YoutubeStreamExtractor) YouTube
+                    .getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetFrames() throws ExtractionException {
+            final List<Frameset> frames = extractor.getFrames();
+            assertNotNull(frames);
+            assertFalse(frames.isEmpty());
+            for (final Frameset f : frames) {
+                for (final String url : f.getUrls()) {
+                    ExtractorAsserts.assertIsValidUrl(url);
+                    ExtractorAsserts.assertIsSecureUrl(url);
+                }
+            }
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -0,0 +1,138 @@
+package org.schabi.newpipe.extractor.services.youtube.stream;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.Localization;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+public class YoutubeStreamExtractorLivestreamTest {
+    private static YoutubeStreamExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
+        extractor = (YoutubeStreamExtractor) YouTube
+                .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void testGetInvalidTimeStamp() throws ParsingException {
+        assertTrue(extractor.getTimeStamp() + "",
+                extractor.getTimeStamp() <= 0);
+    }
+
+    @Test
+    public void testGetTitle() throws ParsingException {
+        assertFalse(extractor.getName().isEmpty());
+    }
+
+    @Test
+    public void testGetDescription() throws ParsingException {
+        assertNotNull(extractor.getDescription());
+        assertFalse(extractor.getDescription().isEmpty());
+    }
+
+    @Test
+    public void testGetFullLinksInDescription() throws ParsingException {
+        assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
+        assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
+    }
+
+    @Test
+    public void testGetUploaderName() throws ParsingException {
+        assertNotNull(extractor.getUploaderName());
+        assertFalse(extractor.getUploaderName().isEmpty());
+    }
+
+
+    @Test
+    public void testGetLength() throws ParsingException {
+        assertEquals(0, extractor.getLength());
+    }
+
+    @Test
+    public void testGetViewCount() throws ParsingException {
+        long count = extractor.getViewCount();
+        assertTrue(Long.toString(count), count >= 7148995);
+    }
+
+    @Test
+    public void testGetUploadDate() throws ParsingException {
+        assertTrue(extractor.getUploadDate().length() > 0);
+    }
+
+    @Test
+    public void testGetUploaderUrl() throws ParsingException {
+        assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
+    }
+
+    @Test
+    public void testGetThumbnailUrl() throws ParsingException {
+        assertIsSecureUrl(extractor.getThumbnailUrl());
+    }
+
+    @Test
+    public void testGetUploaderAvatarUrl() throws ParsingException {
+        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
+    }
+
+    @Test
+    public void testGetAudioStreams() throws ExtractionException {
+        assertFalse(extractor.getAudioStreams().isEmpty());
+    }
+
+    @Test
+    public void testGetVideoStreams() throws ExtractionException {
+        for (VideoStream s : extractor.getVideoStreams()) {
+            assertIsSecureUrl(s.url);
+            assertTrue(s.resolution.length() > 0);
+            assertTrue(Integer.toString(s.getFormatId()),
+                    0 <= s.getFormatId() && s.getFormatId() <= 0x100);
+        }
+    }
+
+    @Test
+    public void testStreamType() throws ParsingException {
+        assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
+    }
+
+    @Test
+    public void testGetDashMpd() throws ParsingException {
+        // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
+        assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty());
+    }
+
+    @Test
+    public void testGetRelatedVideos() throws ExtractionException, IOException {
+        StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
+        Utils.printErrors(relatedVideos.getErrors());
+        assertFalse(relatedVideos.getItems().isEmpty());
+        assertTrue(relatedVideos.getErrors().isEmpty());
+    }
+
+    @Test
+    public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
+        assertTrue(extractor.getSubtitlesDefault().isEmpty());
+    }
+
+    @Test
+    public void testGetSubtitlesList() throws IOException, ExtractionException {
+        assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java

@@ -0,0 +1,18 @@
+package org.schabi.newpipe.extractor.utils;
+
+import com.grack.nanojson.JsonParserException;
+import org.junit.Test;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import static org.junit.Assert.assertEquals;
+
+public class UtilsTest {
+    @Test
+    public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
+        assertEquals(10, Utils.mixedNumberWordToLong("10"));
+        assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
+        assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
+        assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
+        assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
+    }
+}

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Fri Jan 18 11:51:40 CET 2019
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip

gradlew

@@ -1,5 +1,21 @@
 #!/usr/bin/env sh
 
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ##############################################################################
 ##
 ##  Gradle start up script for UN*X
@@ -28,7 +44,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"

gradlew.bat

@@ -1,3 +1,19 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
@@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome