Merge branch 'dev' into bandcamp

.github/workflows/ci.yml

@@ -1,6 +1,9 @@
 name: CI
 
 on:
+  schedule:
+      # once per day
+    - cron: 0 0 * * *
   push:
     branches:
       - dev
@@ -25,5 +28,11 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
           restore-keys: ${{ runner.os }}-gradle
 
+        # See gradle file for difference between downloaders
       - name: Build and run Tests
-        run: ./gradlew check --stacktrace -Ddownloader=MOCK
+        run: |
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            ./gradlew check --stacktrace -Ddownloader=REAL
+          else
+            ./gradlew check --stacktrace -Ddownloader=MOCK
+          fi

README.md

@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
 If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
 
 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
-2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.8'`the `dependencies` in your `build.gradle`. Replace `v0.20.8` with the latest release.
+2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.10'`the `dependencies` in your `build.gradle`. Replace `v0.20.10` with the latest release.
 
 **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
 

build.gradle

@@ -5,7 +5,7 @@ allprojects {
     sourceCompatibility = 1.8
     targetCompatibility = 1.8
 
-    version 'v0.20.8'
+    version 'v0.20.10'
     group 'com.github.TeamNewPipe'
 
     repositories {

extractor/build.gradle

@@ -15,6 +15,6 @@ dependencies {
     implementation 'org.nibor.autolink:autolink:0.10.0'
 
     testImplementation 'junit:junit:4.13.1'
-    testImplementation "com.squareup.okhttp3:okhttp:3.12.11"
+    testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
     testImplementation 'com.google.code.gson:gson:2.8.6'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java

@@ -25,7 +25,7 @@ public class CommentsInfo extends ListInfo<CommentsInfoItem> {
         return getInfo(serviceByUrl.getCommentsExtractor(url));
     }
 
-    private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
+    public static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
         // for services which do not have a comments extractor
         if (null == commentsExtractor) {
             return null;

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java

@@ -17,6 +17,7 @@ public class CommentsInfoItem extends InfoItem {
     private DateWrapper uploadDate;
     private int likeCount;
     private boolean heartedByUploader;
+    private boolean pinned;
 
     public CommentsInfoItem(int serviceId, String url, String name) {
         super(InfoType.COMMENT, serviceId, url, name);
@@ -91,7 +92,15 @@ public class CommentsInfoItem extends InfoItem {
         this.heartedByUploader = isHeartedByUploader;
     }
 
-    public boolean getHeartedByUploader() {
+    public boolean isHeartedByUploader() {
         return this.heartedByUploader;
     }
+
+    public boolean isPinned() {
+        return pinned;
+    }
+
+    public void setPinned(boolean pinned) {
+        this.pinned = pinned;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java

@@ -11,6 +11,7 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Return the like count of the comment, or -1 if it's unavailable
+     *
      * @see StreamExtractor#getLikeCount()
      */
     int getLikeCount() throws ParsingException;
@@ -22,12 +23,14 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * The upload date given by the service, unmodified
+     *
      * @see StreamExtractor#getTextualUploadDate()
      */
     String getTextualUploadDate() throws ParsingException;
 
     /**
      * The upload date wrapped with DateWrapper class
+     *
      * @see StreamExtractor#getUploadDate()
      */
     @Nullable
@@ -44,5 +47,10 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
     /**
      * Whether the comment has been hearted by the uploader
      */
-    boolean getHeartedByUploader() throws ParsingException;
+    boolean isHeartedByUploader() throws ParsingException;
+
+    /**
+     * Whether the comment is pinned
+     */
+    boolean isPinned() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java

@@ -71,7 +71,13 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
         }
 
         try {
-            resultItem.setHeartedByUploader(extractor.getHeartedByUploader());
+            resultItem.setHeartedByUploader(extractor.isHeartedByUploader());
+        } catch (Exception e) {
+            addError(e);
+        }
+
+        try {
+            resultItem.setPinned(extractor.isPinned());
         } catch (Exception e) {
             addError(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -14,6 +14,8 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 public class KioskList {
 
     public interface KioskExtractorFactory {
@@ -70,7 +72,7 @@ public class KioskList {
 
     public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
             throws ExtractionException, IOException {
-        if (defaultKiosk != null && !defaultKiosk.equals("")) {
+        if (!isNullOrEmpty(defaultKiosk)) {
             return getExtractorById(defaultKiosk, nextPage, localization);
         } else {
             if (!kioskList.isEmpty()) {

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -50,6 +50,9 @@ public abstract class LinkHandlerFactory {
      * @return a {@link LinkHandler} complete with information
      */
     public LinkHandler fromUrl(final String url) throws ParsingException {
+        if (Utils.isNullOrEmpty(url)) {
+            throw new IllegalArgumentException("The url is null or empty");
+        }
         final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
         final String baseUrl = Utils.getBaseUrl(polishedUrl);
         return fromUrl(polishedUrl, baseUrl);

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java

@@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
 public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
 
     ///////////////////////////////////
@@ -12,7 +14,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
     ///////////////////////////////////
 
     @Override
-    public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
+    public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) throws ParsingException;
 
     public String getSearchString(String url) {
         return "";
@@ -28,14 +30,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
     }
 
     @Override
-    public SearchQueryHandler fromQuery(String querry,
+    public SearchQueryHandler fromQuery(String query,
                                         List<String> contentFilter,
                                         String sortFilter) throws ParsingException {
-        return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter));
+        return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter));
     }
 
-    public SearchQueryHandler fromQuery(String querry) throws ParsingException {
+    public SearchQueryHandler fromQuery(String query) throws ParsingException {
-        return fromQuery(querry, new ArrayList<String>(0), "");
+        return fromQuery(query, new ArrayList<String>(0), EMPTY_STRING);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java

@@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
     public static final String ALL = "all";
     public static final String CONFERENCES = "conferences";
@@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
                          final String sortFilter) throws ParsingException {
         try {
             return "https://media.ccc.de/public/events/search?q="
-                    + URLEncoder.encode(query, "UTF-8");
+                    + URLEncoder.encode(query, UTF_8);
         } catch (UnsupportedEncodingException e) {
             throw new ParsingException("Could not create search string with query: " + query, e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
 import com.grack.nanojson.JsonObject;
-
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.ServiceList;
@@ -13,6 +12,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.util.Objects;
 
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
 public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
     private final JsonObject item;
     private final String url;
@@ -68,7 +69,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
             final Document doc = Jsoup.parse(htmlText);
             return doc.body().text();
         } catch (Exception e) {
-            return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
+            return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", EMPTY_STRING);
         }
     }
 
@@ -89,7 +90,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
     }
 
     @Override
-    public boolean getHeartedByUploader() throws ParsingException {
+    public boolean isHeartedByUploader() throws ParsingException {
+        return false;
+    }
+
+    @Override
+    public boolean isPinned() throws ParsingException {
         return false;
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.Description;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamSegment;
-import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.SubtitlesStream;
-import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -40,6 +32,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 public class PeertubeStreamExtractor extends StreamExtractor {
     private final String baseUrl;
     private JsonObject json;
@@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         params.append("start=0&count=8&sort=-createdAt");
         for (final String tag : tags) {
             params.append("&tagsOneOf=");
-            params.append(URLEncoder.encode(tag, "UTF-8"));
+            params.append(URLEncoder.encode(tag, UTF_8));
         }
         return url + "?" + params.toString();
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java

@@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
 
-    public static final String CHARSET_UTF_8 = "UTF-8";
     public static final String VIDEOS = "videos";
     public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
     public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
@@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
     public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
         try {
             final String url = baseUrl + SEARCH_ENDPOINT
-                    + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
+                    + "?search=" + URLEncoder.encode(searchString, UTF_8);
 
             return url;
         } catch (UnsupportedEncodingException e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -37,9 +37,7 @@ import java.util.List;
 
 import static java.util.Collections.singletonList;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.*;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudParsingHelper {
     private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
@@ -117,7 +115,7 @@ public class SoundcloudParsingHelper {
      */
     public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
         String apiUrl = "https://api-v2.soundcloud.com/resolve"
-                + "?url=" + URLEncoder.encode(url, "UTF-8")
+                + "?url=" + URLEncoder.encode(url, UTF_8)
                 + "&client_id=" + clientId();
 
         try {
@@ -136,7 +134,7 @@ public class SoundcloudParsingHelper {
     public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
 
         String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+                + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
 
         return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
     }
@@ -148,17 +146,17 @@ public class SoundcloudParsingHelper {
      */
     public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
         // Remove the tailing slash from URLs due to issues with the SoundCloud API
-        if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
+        if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
 
         URL url;
         try {
             url = Utils.stringToURL(urlString);
-        } catch (MalformedURLException e){
+        } catch (MalformedURLException e) {
             throw new IllegalArgumentException("The given URL is not valid");
         }
 
         String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
+                + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
         // handle playlists / sets different and get playlist id via uir field in JSON
         if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
             return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
@@ -240,10 +238,14 @@ public class SoundcloudParsingHelper {
      * @return the next streams url, empty if don't have
      */
     public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
-        String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
+        final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
+        if (response.responseCode() >= 400) {
+            throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
+        }
+
         JsonObject responseObject;
         try {
-            responseObject = JsonParser.object().from(response);
+            responseObject = JsonParser.object().from(response.responseBody());
         } catch (JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -15,11 +14,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import javax.annotation.Nonnull;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 @SuppressWarnings("WeakerAccess")

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java

@@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
 import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java

@@ -6,13 +6,13 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import java.io.IOException;
-
 import javax.annotation.Nonnull;
+import java.io.IOException;
 
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -61,10 +61,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
             apiUrl += "&kind=trending";
         }
 
-        final String contentCountry = SoundCloud.getContentCountry().getCountryCode();
+        final ContentCountry contentCountry = SoundCloud.getContentCountry();
-        apiUrl += "&region=soundcloud:regions:" + contentCountry;
+        String apiUrlWithRegion = null;
+        if (getService().getSupportedCountries().contains(contentCountry)) {
+            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
+        }
 
-        final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+        String nextPageUrl;
+        try {
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
+        } catch (IOException e) {
+            // Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
+            // we retry without the specified region.
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+        }
 
         return new InfoItemsPage<>(collector, new Page(nextPageUrl));
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java

@@ -40,7 +40,12 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
     }
 
     @Override
-    public boolean getHeartedByUploader() throws ParsingException {
+    public boolean isHeartedByUploader() throws ParsingException {
+        return false;
+    }
+
+    @Override
+    public boolean isPinned() throws ParsingException {
         return false;
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java

@@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
 
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java

@@ -4,13 +4,7 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
+import org.schabi.newpipe.extractor.*;
-import org.schabi.newpipe.extractor.InfoItem;
-import org.schabi.newpipe.extractor.InfoItemExtractor;
-import org.schabi.newpipe.extractor.InfoItemsCollector;
-import org.schabi.newpipe.extractor.MetaInfo;
-import org.schabi.newpipe.extractor.Page;
-import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -19,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
@@ -26,10 +21,8 @@ import java.net.URL;
 import java.util.Collections;
 import java.util.List;
 
-import javax.annotation.Nonnull;
-
 import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class SoundcloudSearchExtractor extends SearchExtractor {
@@ -50,7 +43,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
         return false;
     }
 
-    @Nonnull 
+    @Nonnull
     @Override
     public List<MetaInfo> getMetaInfo() {
         return Collections.emptyList();

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.Description;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamSegment;
-import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.SubtitlesStream;
-import org.schabi.newpipe.extractor.stream.VideoStream;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
@@ -34,11 +28,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
-import javax.annotation.Nonnull;
+import static org.schabi.newpipe.extractor.utils.Utils.*;
-import javax.annotation.Nullable;
-
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
@@ -60,7 +50,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getId() {
-        return track.getInt("id") + "";
+        return track.getInt("id") + EMPTY_STRING;
     }
 
     @Nonnull
@@ -73,8 +63,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Override
     public String getTextualUploadDate() {
         return track.getString("created_at")
-                .replace("T"," ")
+                .replace("T", " ")
-                .replace("Z", "");
+                .replace("Z", EMPTY_STRING);
     }
 
     @Nonnull
@@ -232,7 +222,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     private static String urlEncode(String value) {
         try {
-            return URLEncoder.encode(value, "UTF-8");
+            return URLEncoder.encode(value, UTF_8);
         } catch (UnsupportedEncodingException e) {
             throw new IllegalStateException(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java

@@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java

@@ -17,9 +17,9 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.List;
 
-public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
-    public static final String CHARSET_UTF_8 = "UTF-8";
+public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
 
     public SoundcloudSuggestionExtractor(StreamingService service) {
         super(service);
@@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
         Downloader dl = NewPipe.getDownloader();
 
         String url = "https://api-v2.soundcloud.com/search/queries"
-                + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+                + "?q=" + URLEncoder.encode(query, UTF_8)
                 + "&client_id=" + SoundcloudParsingHelper.clientId()
                 + "&limit=10";
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java

@@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
-    public static final String CHARSET_UTF_8 = "UTF-8";
 
     public static final String TRACKS = "tracks";
     public static final String USERS = "users";
@@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
                 }
             }
 
-            return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8)
+            return url + "?q=" + URLEncoder.encode(id, UTF_8)
                     + "&client_id=" + SoundcloudParsingHelper.clientId()
                     + "&limit=" + ITEMS_PER_PAGE
                     + "&offset=0";

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1,14 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import com.grack.nanojson.*;
-import com.grack.nanojson.JsonArray;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
-import com.grack.nanojson.JsonWriter;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -21,27 +13,27 @@ import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeParseException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.*;
-import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
-import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.join;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -111,7 +103,7 @@ public class YoutubeParsingHelper {
         return host.equalsIgnoreCase("invidio.us")
                 || host.equalsIgnoreCase("dev.invidio.us")
                 || host.equalsIgnoreCase("www.invidio.us")
-                || host.equalsIgnoreCase("vid.encryptionin.space")
+                || host.equalsIgnoreCase("redirect.invidious.io")
                 || host.equalsIgnoreCase("invidious.snopyta.org")
                 || host.equalsIgnoreCase("yewtu.be")
                 || host.equalsIgnoreCase("tube.connect.cafe")
@@ -122,11 +114,16 @@ public class YoutubeParsingHelper {
                 || host.equalsIgnoreCase("invidious.xyz")
                 || host.equalsIgnoreCase("vid.mint.lgbt")
                 || host.equalsIgnoreCase("invidiou.site")
-                || host.equalsIgnoreCase("invidious.fdn.fr");
+                || host.equalsIgnoreCase("invidious.fdn.fr")
+                || host.equalsIgnoreCase("invidious.048596.xyz")
+                || host.equalsIgnoreCase("invidious.zee.li")
+                || host.equalsIgnoreCase("vid.puffyan.us")
+                || host.equalsIgnoreCase("ytprivate.com");
     }
 
     /**
      * Parses the duration string of the video expecting ":" or "." as separators
+     *
      * @return the duration in seconds
      * @throws ParsingException when more than 3 separators are found
      */
@@ -196,6 +193,7 @@ public class YoutubeParsingHelper {
     /**
      * Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
      * Ids from a YouTube Mix start with "RD"
+     *
      * @param playlistId
      * @return Whether given id belongs to a YouTube Mix
      */
@@ -206,15 +204,18 @@ public class YoutubeParsingHelper {
     /**
      * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
      * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
+     *
      * @param playlistId
      * @return Whether given id belongs to a YouTube Music Mix
      */
     public static boolean isYoutubeMusicMixId(final String playlistId) {
         return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
     }
+
     /**
      * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
      * Ids from a YouTube channel Mix start with "RDCM"
+     *
      * @return Whether given id belongs to a YouTube Channel Mix
      */
     public static boolean isYoutubeChannelMixId(final String playlistId) {
@@ -223,6 +224,7 @@ public class YoutubeParsingHelper {
 
     /**
      * Extracts the video id from the playlist id for Mixes.
+     *
      * @throws ParsingException If the playlistId is a Channel Mix or not a mix.
      */
     public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
@@ -314,7 +316,8 @@ public class YoutubeParsingHelper {
                     clientVersion = contextClientVersion;
                     break;
                 }
-            } catch (Parser.RegexException ignored) { }
+            } catch (Parser.RegexException ignored) {
+            }
         }
 
         if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
@@ -326,7 +329,8 @@ public class YoutubeParsingHelper {
         } catch (Parser.RegexException e) {
             try {
                 key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
-            } catch (Parser.RegexException ignored) { }
+            } catch (Parser.RegexException ignored) {
+            }
         }
     }
 
@@ -358,7 +362,7 @@ public class YoutubeParsingHelper {
      *
      * Quick-and-dirty solution to reset global state in between test classes.
      */
-    static void resetClientVersionAndKey() {
+    public static void resetClientVersionAndKey() {
         clientVersion = null;
         key = null;
     }
@@ -393,7 +397,7 @@ public class YoutubeParsingHelper {
                 .end()
                 .value("query", "test")
                 .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
-            .end().done().getBytes("UTF-8");
+            .end().done().getBytes(UTF_8);
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
@@ -438,10 +442,17 @@ public class YoutubeParsingHelper {
         return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
     }
 
+
+
     @Nullable
     public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
         if (navigationEndpoint.has("urlEndpoint")) {
             String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
+            if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
+                // remove https://www.youtube.com part to fall in the next if block
+                internUrl = internUrl.substring(23);
+            }
+
             if (internUrl.startsWith("/redirect?")) {
                 // q parameter can be the first parameter
                 internUrl = internUrl.substring(10);
@@ -450,7 +461,7 @@ public class YoutubeParsingHelper {
                     if (param.split("=")[0].equals("q")) {
                         String url;
                         try {
-                            url = URLDecoder.decode(param.split("=")[1], "UTF-8");
+                            url = URLDecoder.decode(param.split("=")[1], UTF_8);
                         } catch (UnsupportedEncodingException e) {
                             return null;
                         }
@@ -459,6 +470,8 @@ public class YoutubeParsingHelper {
                 }
             } else if (internUrl.startsWith("http")) {
                 return internUrl;
+            } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) {
+                return "https://www.youtube.com" + internUrl;
             }
         } else if (navigationEndpoint.has("browseEndpoint")) {
             final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
@@ -496,6 +509,7 @@ public class YoutubeParsingHelper {
 
     /**
      * Get the text from a JSON object that has either a simpleText or a runs array.
+     *
      * @param textObject JSON object to get the text from
      * @param html       whether to return HTML, by parsing the navigationEndpoint
      * @return text in the JSON object or {@code null}
@@ -723,7 +737,7 @@ public class YoutubeParsingHelper {
 
         final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
         final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
-        if (title == null || text ==  null) {
+        if (title == null || text == null) {
             throw new ParsingException("Could not extract clarification renderer content");
         }
         metaInfo.setTitle(title);
@@ -767,6 +781,7 @@ public class YoutubeParsingHelper {
     /**
      * Sometimes, YouTube provides URLs which use Google's cache. They look like
      * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
+     *
      * @param url the URL which might refer to the Google's webcache
      * @return the URL which is referring to the original site
      */

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -20,10 +20,8 @@ import org.schabi.newpipe.extractor.utils.Utils;
 import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*
@@ -230,9 +228,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                     .getArray("contents").getObject(0).getObject("itemSectionRenderer")
                     .getArray("contents").getObject(0).getObject("gridRenderer");
 
-            collectStreamsFrom(collector, gridRenderer.getArray("items"));
+            final JsonObject continuation = collectStreamsFrom(collector, gridRenderer.getArray("items"));
 
-            nextPage = getNextPageFrom(gridRenderer.getArray("continuations"));
+            nextPage = getNextPageFrom(continuation);
         }
 
         return new InfoItemsPage<>(collector, nextPage);
@@ -252,36 +250,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
 
         JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
-                .getObject("continuationContents").getObject("gridContinuation");
+                .getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
 
-        collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
+        final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
 
-        return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
+        return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
     }
 
-    private Page getNextPageFrom(final JsonArray continuations) {
+    private Page getNextPageFrom(final JsonObject continuations) {
         if (isNullOrEmpty(continuations)) {
             return null;
         }
 
-        final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
+        final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
-        final String continuation = nextContinuationData.getString("continuation");
+        final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
-        final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
+        final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
         return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
                 + "&continuation=" + continuation + "&itct=" + clickTrackingParams);
     }
 
-    private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
+    /**
+     * Collect streams from an array of items
+     *
+     * @param collector the collector where videos will be commited
+     * @param videos    the array to get videos from
+     * @return the continuation object
+     * @throws ParsingException if an error happened while extracting
+     */
+    private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
         collector.reset();
 
         final String uploaderName = getName();
         final String uploaderUrl = getUrl();
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        for (Object video : videos) {
+        JsonObject continuation = null;
-            if (((JsonObject) video).has("gridVideoRenderer")) {
+
+        for (Object object : videos) {
+            final JsonObject video = (JsonObject) object;
+            if (video.has("gridVideoRenderer")) {
                 collector.commit(new YoutubeStreamInfoItemExtractor(
-                        ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
+                        video.getObject("gridVideoRenderer"), timeAgoParser) {
                     @Override
                     public String getUploaderName() {
                         return uploaderName;
@@ -292,8 +301,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                         return uploaderUrl;
                     }
                 });
+            } else if (video.has("continuationItemRenderer")) {
+                continuation = video.getObject("continuationItemRenderer");
             }
         }
+
+        return continuation;
     }
 
     private JsonObject getVideoTab() throws ParsingException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
@@ -27,9 +27,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
-import javax.annotation.Nonnull;
-
 import static java.util.Collections.singletonList;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class YoutubeCommentsExtractor extends CommentsExtractor {
@@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
                 first = false;
             else
                 result.append("&");
-            result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
+            result.append(URLEncoder.encode(entry.getKey(), UTF_8));
             result.append("=");
-            result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
+            result.append(URLEncoder.encode(entry.getValue(), UTF_8));
         }
         return result.toString();
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -13,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
 
@@ -46,7 +46,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         try {
             return getTextFromObject(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            return "";
+            return EMPTY_STRING;
         }
     }
 
@@ -86,7 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
             if (contentText.isEmpty()) {
                 // completely empty comments as described in
                 // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
-                return "";
+                return EMPTY_STRING;
             }
             final String commentText = getTextFromObject(contentText);
             // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
@@ -116,16 +116,21 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     }
 
     @Override
-    public boolean getHeartedByUploader() throws ParsingException {
+    public boolean isHeartedByUploader() throws ParsingException {
         return json.has("creatorHeart");
     }
 
+    @Override
+    public boolean isPinned() {
+        return json.has("pinnedCommentBadge");
+    }
+
     @Override
     public String getUploaderName() throws ParsingException {
         try {
             return getTextFromObject(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            return "";
+            return EMPTY_STRING;
         }
     }
 
@@ -134,7 +139,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         try {
             return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
         } catch (Exception e) {
-            return "";
+            return EMPTY_STRING;
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -1,11 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.*;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
-import com.grack.nanojson.JsonWriter;
-
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
@@ -20,27 +15,20 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
+import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import javax.annotation.Nonnull;
-
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
+import static org.schabi.newpipe.extractor.utils.Utils.*;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class YoutubeMusicSearchExtractor extends SearchExtractor {
     private JsonObject initialData;
@@ -105,7 +93,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                 .end()
                 .value("query", getSearchString())
                 .value("params", params)
-            .end().done().getBytes("UTF-8");
+            .end().done().getBytes(UTF_8);
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
@@ -229,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                         .value("enableSafetyMode", false)
                     .end()
                 .end()
-            .end().done().getBytes("UTF-8");
+            .end().done().getBytes(UTF_8);
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
@@ -365,7 +353,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                                     .getObject(descriptionElements.size() - 3)
                                     .getString("text");
                             if (!isNullOrEmpty(viewCount)) {
-                                return Utils.mixedNumberWordToLong(viewCount);
+                                try {
+                                    return Utils.mixedNumberWordToLong(viewCount);
+                                } catch (final Parser.RegexException e) {
+                                    // probably viewCount == "No views" or similar
+                                    return 0;
+                                }
                             }
                             throw new ParsingException("Could not get view count");
                         }
@@ -421,10 +414,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
 
                         @Override
                         public long getSubscriberCount() throws ParsingException {
-                            final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
+                            final String subscriberCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
                                     .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
-                            if (!isNullOrEmpty(viewCount)) {
+                            if (!isNullOrEmpty(subscriberCount)) {
-                                return Utils.mixedNumberWordToLong(viewCount);
+                                try {
+                                    return Utils.mixedNumberWordToLong(subscriberCount);
+                                } catch (final Parser.RegexException ignored) {
+                                    // probably subscriberCount == "No subscribers" or similar
+                                    return 0;
+                                }
                             }
                             throw new ParsingException("Could not get subscriber count");
                         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -82,7 +82,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getName() throws ParsingException {
         final String name = getTextFromObject(playlistInfo.getObject("title"));
-        if (name != null && !name.isEmpty()) return name;
+        if (!isNullOrEmpty(name)) return name;
 
         return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -1,11 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.*;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
-import com.grack.nanojson.JsonWriter;
-
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
@@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import java.io.IOException;
-import java.util.*;
-
 import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*
@@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
                         .object("user").end()
                     .end()
                     .value("continuation", page.getId())
-                .end().done().getBytes("UTF-8");
+                .end().done().getBytes(UTF_8);
             // @formatter:on
 
             final Map<String, List<String>> headers = new HashMap<>();

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -29,17 +28,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.Description;
-import org.schabi.newpipe.extractor.stream.Frameset;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamSegment;
-import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.SubtitlesStream;
-import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -47,26 +36,13 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
-import java.io.UnsupportedEncodingException;
+import java.util.*;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*
@@ -102,17 +78,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     /*//////////////////////////////////////////////////////////////////////////*/
 
-    @Nullable private static String cachedDeobfuscationCode = null;
+    @Nullable
-    @Nullable private String playerJsUrl = null;
+    private static String cachedDeobfuscationCode = null;
+    @Nullable
+    private String playerJsUrl = null;
 
     private JsonArray initialAjaxJson;
     private JsonObject initialData;
-    @Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
+    @Nonnull
+    private final Map<String, String> videoInfoPage = new HashMap<>();
     private JsonObject playerResponse;
     private JsonObject videoPrimaryInfoRenderer;
     private JsonObject videoSecondaryInfoRenderer;
     private int ageLimit = -1;
-    @Nullable private List<SubtitlesStream> subtitles = null;
+    @Nullable
+    private List<SubtitlesStream> subtitles = null;
 
     public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
         super(service, linkHandler);
@@ -224,7 +204,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public Description getDescription() {
+    public Description getDescription() throws ParsingException {
         assertPageFetched();
         // description with more info on links
         try {
@@ -234,8 +214,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             // age-restricted videos cause a ParsingException here
         }
 
+        String description = playerResponse.getObject("videoDetails").getString("shortDescription");
+        if (description == null) {
+            final JsonObject descriptionObject = playerResponse.getObject("microformat")
+                    .getObject("playerMicroformatRenderer").getObject("description");
+            description = getTextFromObject(descriptionObject);
+        }
+
         // raw non-html description
-        return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
+        return new Description(description, Description.PLAIN_TEXT);
     }
 
     @Override
@@ -780,8 +767,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
 
-
-
     private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
         Parser.RegexException exception = null;
         for (final String regex : REGEXES) {
@@ -1079,8 +1064,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             // Search for correct panel containing the data
             for (int i = 0; i < panels.size(); i++) {
-                if (panels.getObject(i).getObject("engagementPanelSectionListRenderer")
+                final String panelIdentifier = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
-                        .getString("panelIdentifier").equals("engagement-panel-macro-markers")) {
+                        .getString("panelIdentifier");
+                if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters")
+                        || panelIdentifier.equals("engagement-panel-macro-markers")) {
                     segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
                             .getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
                     break;
@@ -1129,6 +1116,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public List<MetaInfo> getMetaInfo() throws ParsingException {
         return YoutubeParsingHelper.getMetaInfo(
                 initialData.getObject("contents").getObject("twoColumnWatchNextResults")
-                .getObject("results").getObject("results").getArray("contents"));
+                        .getObject("results").getObject("results").getArray("contents"));
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -17,10 +17,8 @@ import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -15,6 +15,8 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 /*
  * Created by Christian Schabesberger on 28.09.16.
  *
@@ -37,8 +39,6 @@ import java.util.List;
 
 public class YoutubeSuggestionExtractor extends SuggestionExtractor {
 
-    public static final String CHARSET_UTF_8 = "UTF-8";
-
     public YoutubeSuggestionExtractor(StreamingService service) {
         super(service);
     }
@@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
                 + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
                 + "&jsonp=" + "JP"
                 + "&ds=" + "yt"
-                + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+                + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
-                + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
+                + "&q=" + URLEncoder.encode(query, UTF_8);
 
         String response = dl.get(url, getExtractorLocalization()).responseBody();
         // trim JSONP part "JP(...)"

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -69,7 +69,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
     @Nonnull
     @Override
     public String getName() throws ParsingException {
-        String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
+        final JsonObject header = initialData.getObject("header");
+        JsonObject title = null;
+        if (header.has("feedTabbedHeaderRenderer")) {
+            title = header.getObject("feedTabbedHeaderRenderer").getObject("title");
+        } else if (header.has("c4TabbedHeaderRenderer")) {
+            title = header.getObject("c4TabbedHeaderRenderer").getObject("title");
+        }
+
+        String name = getTextFromObject(title);
         if (!isNullOrEmpty(name)) {
             return name;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java

@@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
-    public static final String CHARSET_UTF_8 = "UTF-8";
 
     public static final String ALL = "all";
     public static final String VIDEOS = "videos";
@@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
                     default:
                         break;
                     case VIDEOS:
-                        return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
+                        return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
                     case CHANNELS:
-                        return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
+                        return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
                     case PLAYLISTS:
-                        return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
+                        return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
                     case MUSIC_SONGS:
                     case MUSIC_VIDEOS:
                     case MUSIC_ALBUMS:
                     case MUSIC_PLAYLISTS:
                     case MUSIC_ARTISTS:
-                        return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
+                        return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
                 }
             }
 
-            return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
+            return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
         } catch (UnsupportedEncodingException e) {
             throw new ParsingException("Could not encode query", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -36,7 +36,7 @@ import java.util.regex.Pattern;
 
 public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
-    private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("([a-zA-Z0-9_-]{11})");
+    private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})");
     private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
 
     private YoutubeStreamLinkHandlerFactory() {
@@ -183,10 +183,10 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
                 // there is no break-statement here on purpose so the next code-block gets also run for hooktube
             }
 
-            case "WWW.INVIDIO.US":
-            case "DEV.INVIDIO.US":
             case "INVIDIO.US":
-            case "VID.ENCRYPTIONIN.SPACE":
+            case "DEV.INVIDIO.US":
+            case "WWW.INVIDIO.US":
+            case "REDIRECT.INVIDIOUS.IO":
             case "INVIDIOUS.SNOPYTA.ORG":
             case "YEWTU.BE":
             case "TUBE.CONNECT.CAFE":
@@ -197,7 +197,11 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             case "INVIDIOUS.XYZ":
             case "VID.MINT.LGBT":
             case "INVIDIOU.SITE":
-            case "INVIDIOUS.FDN.FR": { // code-block for hooktube.com and Invidious instances
+            case "INVIDIOUS.FDN.FR":
+            case "INVIDIOUS.048596.XYZ":
+            case "INVIDIOUS.ZEE.LI":
+            case "VID.PUFFYAN.US":
+            case "YTPRIVATE.COM": { // code-block for hooktube.com and Invidious instances
                 if (path.equals("watch")) {
                     String viewQueryValue = Utils.getQueryValue(url, "v");
                     if (viewQueryValue != null) {

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java

@@ -1,13 +1,16 @@
 package org.schabi.newpipe.extractor.stream;
 
 import java.io.Serializable;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 public class Description implements Serializable {
 
     public static final int HTML = 1;
     public static final int MARKDOWN = 2;
     public static final int PLAIN_TEXT = 3;
-    public static final Description emptyDescription = new Description("", PLAIN_TEXT);
+    public static final Description emptyDescription = new Description(EMPTY_STRING, PLAIN_TEXT);
 
     private String content;
     private int type;
@@ -28,4 +31,17 @@ public class Description implements Serializable {
     public int getType() {
         return type;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Description that = (Description) o;
+        return type == that.type && Objects.equals(content, that.content);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(content, type);
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -64,7 +64,7 @@ public class StreamInfo extends Info {
         return getInfo(service.getStreamExtractor(url));
     }
 
-    private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
+    public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
         extractor.fetchPage();
         StreamInfo streamInfo;
         try {

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -13,7 +13,6 @@ import java.util.List;
 public class JsonUtils {
     public static final JsonObject EMPTY_OBJECT = new JsonObject();
     public static final JsonArray EMPTY_ARRAY = new JsonArray();
-    public static final String EMPTY_STRING = "";
 
     private JsonUtils() {
     }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -14,6 +14,8 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+
 /*
  * Created by Christian Schabesberger on 02.02.16.
  *
@@ -69,9 +71,9 @@ public class Parser {
         } else {
             // only pass input to exception message when it is not too long
             if (input.length() > 1024) {
-                throw new RegexException("failed to find pattern \"" + pat.pattern());
+                throw new RegexException("failed to find pattern \"" + pat.pattern() + "\"");
             } else {
-                throw new RegexException("failed to find pattern \"" + pat.pattern() + " inside of " + input + "\"");
+                throw new RegexException("failed to find pattern \"" + pat.pattern() + "\" inside of \"" + input + "\"");
             }
         }
     }
@@ -87,7 +89,7 @@ public class Parser {
         for (String arg : input.split("&")) {
             String[] splitArg = arg.split("=");
             if (splitArg.length > 1) {
-                map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
+                map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
             } else {
                 map.put(splitArg[0], "");
             }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -6,16 +6,14 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.util.Collection;
+import java.util.*;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 
 public class Utils {
 
     public static final String HTTP = "http://";
     public static final String HTTPS = "https://";
+    public static final String UTF_8 = "UTF-8";
+    public static final String EMPTY_STRING = "";
 
     private Utils() {
         //no instance
@@ -117,7 +115,7 @@ public class Utils {
 
                 String query;
                 try {
-                    query = URLDecoder.decode(params[0], "UTF-8");
+                    query = URLDecoder.decode(params[0], UTF_8);
                 } catch (UnsupportedEncodingException e) {
                     System.err.println("Cannot decode string with UTF-8. using the string without decoding");
                     e.printStackTrace();
@@ -126,7 +124,7 @@ public class Utils {
 
                 if (query.equals(parameterName)) {
                     try {
-                        return URLDecoder.decode(params[1], "UTF-8");
+                        return URLDecoder.decode(params[1], UTF_8);
                     } catch (UnsupportedEncodingException e) {
                         System.err.println("Cannot decode string with UTF-8. using the string without decoding");
                         e.printStackTrace();
@@ -200,6 +198,7 @@ public class Utils {
     /**
      * If the provided url is a Google search redirect, then the actual url is extracted from the
      * {@code url=} query value and returned, otherwise the original url is returned.
+     *
      * @param url the url which can possibly be a Google search redirect
      * @return an url with no Google search redirects
      */
@@ -208,7 +207,7 @@ public class Utils {
         try {
             final URL decoded = Utils.stringToURL(url);
             if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
-                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
+                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
             }
         } catch (final Exception ignored) {
         }
@@ -231,12 +230,12 @@ public class Utils {
         return map == null || map.isEmpty();
     }
 
-    public static boolean isWhitespace(final int c){
+    public static boolean isWhitespace(final int c) {
         return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
     }
 
     public static boolean isBlank(final String string) {
-        if (string == null || string.isEmpty()) {
+        if (isNullOrEmpty(string)) {
             return true;
         }
 
@@ -263,10 +262,10 @@ public class Utils {
     }
 
     public static String join(final String delimiter, final String mapJoin,
-        final Map<? extends CharSequence, ? extends CharSequence> elements) {
+                              final Map<? extends CharSequence, ? extends CharSequence> elements) {
         final List<String> list = new LinkedList<>();
         for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
-            .entrySet()) {
+                .entrySet()) {
             list.add(entry.getKey() + mapJoin + entry.getValue());
         }
         return join(delimiter, list);

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java

@@ -26,16 +26,13 @@ public class MediaCCCRecentListExtractorTest {
     }
 
     @Test
-    @Ignore("TODO fix")
     public void testStreamList() throws Exception {
         final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
         assertEquals(100, items.size());
         for (final StreamInfoItem item: items) {
             assertFalse(isNullOrEmpty(item.getName()));
             assertTrue(item.getDuration() > 0);
-            assertTrue(isNullOrEmpty(item.getUploaderName())); // we do not get the uploader name
             assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
-            assertTrue(item.getUploadDate().offsetDateTime().isAfter(OffsetDateTime.now().minusYears(1)));
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java

@@ -101,7 +101,6 @@ public class PeertubeAccountExtractorTest {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testSubscriberCount() throws ParsingException {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java

@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.ExtractorAsserts;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
@@ -51,9 +52,8 @@ public class PeertubePlaylistExtractorTest {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetStreamCount() throws ParsingException {
-            assertEquals(35, extractor.getStreamCount());
+            ExtractorAsserts.assertAtLeast(39, extractor.getStreamCount());
         }
 
         @Test
@@ -62,7 +62,6 @@ public class PeertubePlaylistExtractorTest {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetSubChannelName() throws ParsingException {
             assertEquals("SHOCKING !", extractor.getSubChannelName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -94,13 +94,6 @@ public class PeertubeStreamExtractorTest {
         @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
         @Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
         @Override public int expectedStreamSegmentsCount() { return 0; }
-
-        @Override
-        @Test
-        @Ignore("TODO fix")
-        public void testSubChannelName() throws Exception {
-            super.testSubChannelName();
-        }
     }
 
     @Ignore("TODO fix")

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -42,7 +42,6 @@ public class SoundcloudPlaylistExtractorTest {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testName() {
             assertEquals("THE PERFECT LUV TAPE®️", extractor.getName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java

@@ -26,7 +26,6 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
     }
 
     @Test(expected = IllegalArgumentException.class)
-    @Ignore("TODO fix")
     public void getIdWithNullAsUrl() throws ParsingException {
         linkHandler.fromUrl(null).getId();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java

@@ -19,6 +19,7 @@ import static java.util.Collections.singletonList;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
 import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
 public class SoundcloudSearchExtractorTest {
 
@@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest {
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public SearchExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return SoundCloud; }
         @Override public String expectedName() { return QUERY; }
@@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest {
         @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
         @Override public String expectedSearchString() { return QUERY; }
         @Nullable @Override public String expectedSearchSuggestion() { return null; }
+        // @formatter:on
     }
 
     public static class Tracks extends DefaultSearchExtractorTest {
@@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest {
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public SearchExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return SoundCloud; }
         @Override public String expectedName() { return QUERY; }
@@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest {
         @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
         @Override public String expectedSearchString() { return QUERY; }
         @Nullable @Override public String expectedSearchSuggestion() { return null; }
-
         @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
+        // @formatter:on
     }
 
     public static class Users extends DefaultSearchExtractorTest {
@@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest {
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public SearchExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return SoundCloud; }
         @Override public String expectedName() { return QUERY; }
@@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest {
         @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
         @Override public String expectedSearchString() { return QUERY; }
         @Nullable @Override public String expectedSearchSuggestion() { return null; }
-
         @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
+        // @formatter:on
     }
 
     public static class Playlists extends DefaultSearchExtractorTest {
@@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest {
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public SearchExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return SoundCloud; }
         @Override public String expectedName() { return QUERY; }
@@ -108,13 +114,12 @@ public class SoundcloudSearchExtractorTest {
         @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
         @Override public String expectedSearchString() { return QUERY; }
         @Nullable @Override public String expectedSearchSuggestion() { return null; }
-
         @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
+        // @formatter:on
     }
 
     public static class PagingTest {
         @Test
-        @Ignore("TODO fix")
         public void duplicatedItemsCheck() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), "");
@@ -129,7 +134,7 @@ public class SoundcloudSearchExtractorTest {
 
     private static String urlEncode(String value) {
         try {
-            return URLEncoder.encode(value, CHARSET_UTF_8);
+            return URLEncoder.encode(value, UTF_8);
         } catch (UnsupportedEncodingException e) {
             throw new RuntimeException(e);
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java

@@ -24,7 +24,6 @@ public class SoundcloudSearchQHTest {
     }
 
     @Test
-    @Ignore("TODO fix")
     public void testRegularValues() throws Exception {
         assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0",
                 removeClientId(SoundCloud.getSearchQHFactory().fromQuery("asdf").getUrl()));

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -12,21 +13,29 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
-import static org.junit.Assert.*;
+import java.io.IOException;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 /**
  * Test for {@link ChannelExtractor}
  */
 public class YoutubeChannelExtractorTest {
 
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
+
     public static class NotAvailable {
         @BeforeClass
-        public static void setUp() {
+        public static void setUp() throws IOException {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
         @Test(expected = ContentNotAvailableException.class)
@@ -46,8 +55,9 @@ public class YoutubeChannelExtractorTest {
 
     public static class NotSupported {
         @BeforeClass
-        public static void setUp() {
+        public static void setUp() throws IOException {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
         }
 
         @Test(expected = ContentNotSupportedException.class)
@@ -63,7 +73,8 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("http://www.youtube.com/user/Gronkh");
             extractor.fetchPage();
@@ -116,8 +127,8 @@ public class YoutubeChannelExtractorTest {
          // ChannelExtractor
          //////////////////////////////////////////////////////////////////////////*/
 
+        @Ignore("TODO fix, character ü makes problems")
         @Test
-        @Ignore("TODO fix")
         public void testDescription() throws Exception {
             assertTrue(extractor.getDescription().contains("Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb"));
         }
@@ -154,7 +165,8 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/Vsauce");
             extractor.fetchPage();
@@ -245,22 +257,13 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
             extractor.fetchPage();
         }
 
-        /*//////////////////////////////////////////////////////////////////////////
-        // Additional Testing
-        //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testGetPageInNewExtractor() throws Exception {
-            final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
-            defaultTestGetPageInNewExtractor(extractor, newExtractor);
-        }
-
         /*//////////////////////////////////////////////////////////////////////////
         // Extractor
         //////////////////////////////////////////////////////////////////////////*/
@@ -342,12 +345,33 @@ public class YoutubeChannelExtractorTest {
         }
     }
 
+    public static class KurzgesagtAdditional {
+
+        private static YoutubeChannelExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            // Test is not deterministic, mocks can't be used
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeChannelExtractor) YouTube
+                    .getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetPageInNewExtractor() throws Exception {
+            final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor);
+        }
+    }
+
     public static class CaptainDisillusion implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
             extractor.fetchPage();
@@ -436,7 +460,8 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");
             extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java

@@ -1,8 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import org.junit.Ignore;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -23,14 +22,15 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
 /**
  * A class that tests multiple channels and ranges of "time ago".
  */
-@Ignore("Should be ran manually from time to time, as it's too time consuming.")
 public class YoutubeChannelLocalizationTest {
-    private static final boolean DEBUG = true;
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
+    private static final boolean DEBUG = false;
     private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
 
     @Test
     public void testAllSupportedLocalizations() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localization"));
 
         testLocalizationsFor("https://www.youtube.com/user/NBCNews");
         testLocalizationsFor("https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -1,8 +1,9 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.Page;
@@ -16,10 +17,15 @@ import org.schabi.newpipe.extractor.utils.Utils;
 import java.io.IOException;
 import java.util.List;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeCommentsExtractorTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/comments/";
+
     /**
      * Test a "normal" YouTube
      */
@@ -30,7 +36,8 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
             extractor.fetchPage();
@@ -117,13 +124,15 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
             extractor.fetchPage();
         }
 
         @Test
+        @Ignore("TODO fix")
         public void testGetCommentsAllData() throws IOException, ExtractionException {
             final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
 
@@ -155,7 +164,8 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
             extractor.fetchPage();
@@ -181,7 +191,7 @@ public class YoutubeCommentsExtractorTest {
                 assertFalse(Utils.isBlank(c.getUrl()));
                 assertFalse(c.getLikeCount() < 0);
                 assertFalse(Utils.isBlank(c.getCommentText()));
-                if (c.getHeartedByUploader()) {
+                if (c.isHeartedByUploader()) {
                     heartedByUploader = true;
                 }
             }
@@ -189,4 +199,41 @@ public class YoutubeCommentsExtractorTest {
 
         }
     }
+
+    public static class Pinned {
+        private final static String url = "https://www.youtube.com/watch?v=bjFtFMilb34";
+        private static YoutubeCommentsExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
+            extractor = (YoutubeCommentsExtractor) YouTube
+                    .getCommentsExtractor(url);
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetCommentsAllData() throws IOException, ExtractionException {
+            final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+            DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+
+            for (CommentsInfoItem c : comments.getItems()) {
+                assertFalse(Utils.isBlank(c.getUploaderUrl()));
+                assertFalse(Utils.isBlank(c.getUploaderName()));
+                assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+                assertFalse(Utils.isBlank(c.getCommentId()));
+                assertFalse(Utils.isBlank(c.getName()));
+                assertFalse(Utils.isBlank(c.getTextualUploadDate()));
+                assertNotNull(c.getUploadDate());
+                assertFalse(Utils.isBlank(c.getThumbnailUrl()));
+                assertFalse(Utils.isBlank(c.getUrl()));
+                assertFalse(c.getLikeCount() < 0);
+                assertFalse(Utils.isBlank(c.getCommentText()));
+            }
+
+            assertTrue("First comment isn't pinned", comments.getItems().get(0).isPinned());
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -2,24 +2,29 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
 import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 public class YoutubeFeedExtractorTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/feed/";
+
     public static class Kurzgesagt implements BaseListExtractorTest {
         private static YoutubeFeedExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
             extractor = (YoutubeFeedExtractor) YouTube
                     .getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");
             extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java

@@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
@@ -14,12 +14,16 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreIte
 import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 public class YoutubeKioskExtractorTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/kiosk/";
+    
     public static class Trending implements BaseListExtractorTest {
         private static YoutubeTrendingExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "trending"));
             extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor();
             extractor.fetchPage();
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -1,11 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
 import org.hamcrest.MatcherAssert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -26,6 +20,12 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractor
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -43,7 +43,7 @@ public class YoutubeMixPlaylistExtractorTest {
     private static final String VIDEO_ID = "_AzeUSL9lZc";
     private static final String VIDEO_TITLE =
             "Most Beautiful And Emotional  Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
-    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/mix/";
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
     private static final Map<String, String> dummyCookie
             = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
 
@@ -134,9 +134,9 @@ public class YoutubeMixPlaylistExtractorTest {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+                            "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
-                        + VIDEO_ID + INDEX);
+                                    + VIDEO_ID + INDEX);
             extractor.fetchPage();
         }
 
@@ -165,8 +165,8 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test
         public void getPage() throws Exception {
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
-                new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+                    new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
-                    + VIDEO_ID + INDEX + PBJ, dummyCookie));
+                            + VIDEO_ID + INDEX + PBJ, dummyCookie));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -204,9 +204,9 @@ public class YoutubeMixPlaylistExtractorTest {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
-                        + VIDEO_ID);
+                                    + VIDEO_ID);
             extractor.fetchPage();
         }
 
@@ -238,11 +238,12 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test
         public void getPage() throws Exception {
             final InfoItemsPage<StreamInfoItem> streams =
-                extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+                    extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
-                    + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
+                            + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
+
         @Test
         public void getContinuations() throws Exception {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
@@ -290,8 +291,8 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test(expected = ExtractionException.class)
         public void invalidVideoId() throws Exception {
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
+                            "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
             extractor.fetchPage();
             extractor.getName();
         }
@@ -309,9 +310,9 @@ public class YoutubeMixPlaylistExtractorTest {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                            "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
-                        + "&list=RDCM" + CHANNEL_ID);
+                                    + "&list=RDCM" + CHANNEL_ID);
             extractor.fetchPage();
         }
 
@@ -339,8 +340,8 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test
         public void getPage() throws Exception {
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
-                new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                    new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
-                    + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
+                            + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java

@@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -13,9 +13,13 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class YoutubeParsingHelperTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/";
+
     @BeforeClass
-    public static void setUp() {
+    public static void setUp() throws IOException {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "youtubeParsingHelper"));
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -6,10 +6,9 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 import org.junit.runners.Suite.SuiteClasses;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
-
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@@ -22,13 +21,18 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTes
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+import java.io.IOException;
+
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 /**
  * Test for {@link YoutubePlaylistExtractor}
@@ -38,10 +42,13 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
     LearningPlaylist.class, ContinuationsTests.class})
 public class YoutubePlaylistExtractorTest {
 
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/playlist/";
+
     public static class NotAvailable {
         @BeforeClass
-        public static void setUp() {
+        public static void setUp() throws IOException {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
         @Test(expected = ContentNotAvailableException.class)
@@ -65,7 +72,8 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "TimelessPopHits"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
             extractor.fetchPage();
@@ -162,7 +170,8 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "huge"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj");
             extractor.fetchPage();
@@ -274,7 +283,8 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "learning"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
             extractor.fetchPage();
@@ -370,8 +380,9 @@ public class YoutubePlaylistExtractorTest {
     public static class ContinuationsTests {
 
         @BeforeClass
-        public static void setUp() {
+        public static void setUp() throws IOException {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "continuations"));
         }
 
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java

@@ -42,6 +42,8 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
         invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
         invalidUrls.add("https://www.youtube.com/");
+        invalidUrls.add("https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ");
+        invalidUrls.add("https://invidio.us/channel/UCBR8-60-B28hp2BmDPdntcQ");
         for (String invalidUrl : invalidUrls) {
             Throwable exception = null;
             try {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java

@@ -12,25 +12,25 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
 
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 import static org.schabi.newpipe.FileUtils.resolveTestResource;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
 /**
  * Test for {@link YoutubeSubscriptionExtractor}
  */
 public class YoutubeSubscriptionExtractorTest {
+
+
     private static YoutubeSubscriptionExtractor subscriptionExtractor;
     private static LinkHandlerFactory urlHandler;
 
     @BeforeClass
     public static void setupClass() {
+        //Doesn't make network requests
         NewPipe.init(DownloaderTestImpl.getInstance());
         subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
         urlHandler = ServiceList.YouTube.getChannelLHFactory();
@@ -53,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
     @Test
     public void testEmptySourceException() throws Exception {
         final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
+                new ByteArrayInputStream("[]".getBytes(UTF_8)));
         assertTrue(items.isEmpty());
     }
 
@@ -61,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
     public void testSubscriptionWithEmptyTitleInSource() throws Exception {
         final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
         final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
+                new ByteArrayInputStream(source.getBytes(UTF_8)));
 
         assertEquals(1, items.size());
         assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@@ -74,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
         final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
                 "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
         final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
+                new ByteArrayInputStream(source.getBytes(UTF_8)));
 
         assertEquals(1, items.size());
         assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@@ -98,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
 
         for (String invalidContent : invalidList) {
             try {
-                byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
+                byte[] bytes = invalidContent.getBytes(UTF_8);
                 subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
                 fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
             } catch (final Exception e) {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java

@@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.localization.Localization;
@@ -37,11 +37,15 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * Test for {@link SuggestionExtractor}
  */
 public class YoutubeSuggestionExtractorTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/suggestions/";
+
     private static SuggestionExtractor suggestionExtractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE"));
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + ""), new Localization("de", "DE"));
         suggestionExtractor = YouTube.getSuggestionExtractor();
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java

@@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.kiosk.KioskInfo;
@@ -36,14 +36,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * Test for {@link KioskInfo}
  */
 public class YoutubeTrendingKioskInfoTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk";
+
     static KioskInfo kioskInfo;
 
     @BeforeClass
     public static void setUp()
             throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
-        StreamingService service = YouTube;
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
-        LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");
+        LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
 
         kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java

@@ -10,12 +10,14 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
 
-import javax.annotation.Nullable;
 import java.net.URLEncoder;
 
+import javax.annotation.Nullable;
+
 import static java.util.Collections.singletonList;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
+// Doesn't work with mocks. Makes request with different `dataToSend` i think
 public class YoutubeMusicSearchExtractorTest {
     public static class MusicSongs extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java

@@ -1,24 +1,26 @@
 package org.schabi.newpipe.extractor.services.youtube.search;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
-import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.MetaInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.Description;
 
-import javax.annotation.Nullable;
-
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import javax.annotation.Nullable;
+
 import static java.util.Collections.singletonList;
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
@@ -29,15 +31,20 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplica
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 public class YoutubeSearchExtractorTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/search/";
+
     public static class All extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;
         private static final String QUERY = "test";
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "all"));
             extractor = YouTube.getSearchExtractor(QUERY);
             extractor.fetchPage();
         }
@@ -58,7 +65,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channel"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
             extractor.fetchPage();
         }
@@ -81,7 +89,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "playlist"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), "");
             extractor.fetchPage();
         }
@@ -104,7 +113,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "videos"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
         }
@@ -128,7 +138,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "suggestions"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
         }
@@ -151,7 +162,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "corrected"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
         }
@@ -174,7 +186,8 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
             extractor = YouTube.getSearchExtractor(QUERY);
             extractor.fetchPage();
         }
@@ -208,7 +221,8 @@ public class YoutubeSearchExtractorTest {
     public static class PagingTest {
         @Test
         public void duplicatedItemsCheck() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "paging"));
             final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), "");
             extractor.fetchPage();
 
@@ -219,14 +233,14 @@ public class YoutubeSearchExtractorTest {
         }
     }
 
-    @Ignore("TODO fix")
     public static class MetaInfoTest extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;
         private static final String QUERY = "Covid";
 
         @Test
         public void clarificationTest() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "metaInfo"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java

@@ -7,6 +7,7 @@ import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 public class YoutubeSearchQHTest {
 
@@ -19,37 +20,37 @@ public class YoutubeSearchQHTest {
         assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
         assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
 
-        assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+        assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
-        assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+        assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
-        assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+        assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
-        assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+        assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
-        assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+        assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
     }
 
     @Test
     public void testGetContentFilter() throws Exception {
         assertEquals(VIDEOS, YouTube.getSearchQHFactory()
-                .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
+                .fromQuery(EMPTY_STRING, asList(new String[]{VIDEOS}), EMPTY_STRING).getContentFilters().get(0));
         assertEquals(CHANNELS, YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
+                .fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getContentFilters().get(0));
 
         assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
+                .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getContentFilters().get(0));
     }
 
     @Test
     public void testWithContentfilter() throws Exception {
         assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl());
+                .fromQuery("asdf", asList(new String[]{VIDEOS}), EMPTY_STRING).getUrl());
         assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl());
+                .fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getUrl());
         assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
+                .fromQuery("asdf", asList(new String[]{PLAYLISTS}), EMPTY_STRING).getUrl());
         assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
+                .fromQuery("asdf", asList(new String[]{"fjiijie"}), EMPTY_STRING).getUrl());
 
         assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
-                .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
+                .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -1,10 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -16,6 +17,7 @@ import javax.annotation.Nullable;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "MmBeUZqv1QA";
     private static final int TIMESTAMP = 196;
     private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
@@ -23,7 +25,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
 
     @BeforeClass
     public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ageRestricted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
 import org.junit.Ignore;
-import org.junit.Test;
+import org.schabi.newpipe.downloader.DownloaderFactory;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
@@ -21,14 +21,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 /**
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
+@Ignore("Video is not available anymore")
 public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "T4XJQO3qol8";
     private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
     private static StreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();
     }
@@ -53,11 +56,4 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
     @Override public long expectedLikeCountAtLeast() { return 13300; }
     @Override public long expectedDislikeCountAtLeast() { return 2600; }
-
-    @Override
-    @Test
-    @Ignore("TODO fix")
-    public void testErrorMessage() throws Exception {
-        super.testErrorMessage();
-    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -3,29 +3,31 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamSegment;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import javax.annotation.Nullable;
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import javax.annotation.Nullable;
+import static org.junit.Assert.assertEquals;
-
+import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 /*
  * Created by Christian Schabesberger on 30.12.15.
@@ -47,12 +49,14 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 public class YoutubeStreamExtractorDefaultTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     static final String BASE_URL = "https://www.youtube.com/watch?v=";
 
     public static class NotAvailable {
         @BeforeClass
-        public static void setUp() {
+        public static void setUp() throws IOException {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
         @Test(expected = ContentNotAvailableException.class)
@@ -78,11 +82,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
@@ -105,9 +111,9 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedLikeCountAtLeast() { return 5212900; }
         @Override public long expectedDislikeCountAtLeast() { return 30600; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
+        // @formatter:on
     }
 
-    @Ignore("TODO fix")
     public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
         private static final String ID = "cV5TjZCJkuA";
         private static final String URL = BASE_URL + ID;
@@ -115,11 +121,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "This Smartphone Changes Everything..."; }
@@ -142,6 +150,14 @@ public class YoutubeStreamExtractorDefaultTest {
         @Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
         @Override public long expectedLikeCountAtLeast() { return 340100; }
         @Override public long expectedDislikeCountAtLeast() { return 18700; }
+        // @formatter:on
+        @Override
+        @Test
+        @Ignore("TODO fix")
+        public void testDescription() throws Exception {
+            super.testDescription();
+        }
+
     }
 
     @Ignore("TODO fix")
@@ -153,11 +169,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
@@ -176,6 +194,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
         @Override public long expectedLikeCountAtLeast() { return -1; }
         @Override public long expectedDislikeCountAtLeast() { return -1; }
+        // @formatter:on
     }
 
     public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
@@ -186,11 +205,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsOstCollection"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
@@ -221,6 +242,7 @@ public class YoutubeStreamExtractorDefaultTest {
             assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
             assertNotNull(segment.getPreviewUrl());
         }
+        // @formatter:on
     }
 
     public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@@ -231,11 +253,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; }
@@ -246,9 +270,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "maiLab"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
-        @Override public List<String> expectedDescriptionContains() {
+        @Override public List<String> expectedDescriptionContains()  {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
-            return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");
-        }
         @Override public long expectedLength() { return 1010; }
         @Override public long expectedViewCountAtLeast() { return 815500; }
         @Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; }
@@ -256,10 +278,10 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedLikeCountAtLeast() { return 48500; }
         @Override public long expectedDislikeCountAtLeast() { return 20000; }
         @Override public boolean expectedHasSubtitles() { return true; }
-
         @Override public int expectedStreamSegmentsCount() { return 7; }
+        // @formatter:on
+
         @Test
-        @Ignore("TODO fix")
         public void testStreamSegment() throws Exception {
             final StreamSegment segment = extractor.getStreamSegments().get(1);
             assertEquals(164, segment.getStartTimeSeconds());
@@ -267,9 +289,15 @@ public class YoutubeStreamExtractorDefaultTest {
             assertEquals(BASE_URL + ID + "?t=164", segment.getUrl());
             assertNotNull(segment.getPreviewUrl());
         }
+
+        @Override
+        @Test
+        @Ignore("encoding problem")
+        public void testName() throws Exception {
+            super.testName();
+        }
     }
 
-    @Ignore("TODO fix")
     public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
         private static final String ID = "q6fgbYWsMgw";
         private static final int TIMESTAMP = 0;
@@ -278,11 +306,13 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
+        // @formatter:off
         @Override public StreamExtractor extractor() { return extractor; }
         @Override public StreamingService expectedService() { return YouTube; }
         @Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
@@ -303,12 +333,27 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedDislikeCountAtLeast() { return 500; }
         @Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
             return Collections.singletonList(new MetaInfo(
-                    "",
+                    EMPTY_STRING,
                     new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
                     Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
                     Collections.singletonList("Wikipedia (German)")
             ));
         }
+        // @formatter:on
+        @Override
+        @Ignore("TODO fix")
+        @Test
+        public void testUploaderName() throws Exception {
+            super.testUploaderName();
+        }
+
+        @Override
+        @Ignore("TODO fix")
+        @Test
+        public void testMetaInfo() throws Exception {
+            super.testMetaInfo();
+        }
+
     }
 
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -1,10 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -16,6 +17,7 @@ import javax.annotation.Nullable;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "5qap5aO4i9A";
     private static final int TIMESTAMP = 1737;
     private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
@@ -23,7 +25,8 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
 
     @BeforeClass
     public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "live"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java

@@ -1,10 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -16,13 +17,15 @@ import javax.annotation.Nullable;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     static final String ID = "udsB8KnIJTg";
     static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
     private static StreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unlisted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();
     }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_2.json

@@ -0,0 +1,69 @@
+{
+  "request": {
+    "httpMethod": "GET",
+    "url": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid",
+    "headers": {
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ],
+      "X-YouTube-Client-Name": [
+        "1"
+      ],
+      "X-YouTube-Client-Version": [
+        "2.20200214.04.00"
+      ]
+    },
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 404,
+    "responseMessage": "",
+    "responseHeaders": {
+      "alt-svc": [
+        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+      ],
+      "cache-control": [
+        "no-cache, no-store, max-age\u003d0, must-revalidate"
+      ],
+      "content-type": [
+        "text/html; charset\u003dutf-8"
+      ],
+      "date": [
+        "Sat, 13 Feb 2021 19:13:39 GMT"
+      ],
+      "expires": [
+        "Mon, 01 Jan 1990 00:00:00 GMT"
+      ],
+      "p3p": [
+        "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
+      ],
+      "pragma": [
+        "no-cache"
+      ],
+      "server": [
+        "ESF"
+      ],
+      "set-cookie": [
+        "YSC\u003dbFLZyqvSsDQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003dYgNWpn3F6Ww; Domain\u003d.youtube.com; Expires\u003dThu, 12-Aug-2021 19:13:39 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+      ],
+      "strict-transport-security": [
+        "max-age\u003d31536000"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"AkiRKVBSwx7zy61zR/BBpg\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
+    "latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
+  }
+}