[Youtube] Add cookies to youtube mix request

This way youtube wont return duplicates when getting more items of the mix (but youtube can also track us)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
+
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
 import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.join;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -61,6 +65,12 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
+    /**
+     * The official youtube app supports intents in this format, where after the ':' is the videoId.
+     * Accordingly there are other apps sharing streams in this format.
+     */
+    public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
+
     private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
     private static String clientVersion;
 
@@ -193,22 +203,22 @@ public class YoutubeParsingHelper {
     }
 
     /**
-     * Checks if the given playlist id is a youtube mix (auto-generated playlist)
+     * Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
-     * Ids from a youtube mix start with "RD"
+     * Ids from a YouTube Mix start with "RD"
      * @param playlistId
-     * @return Whether given id belongs to a youtube mix
+     * @return Whether given id belongs to a YouTube Mix
      */
-    public static boolean isYoutubeMixId(String playlistId) {
+    public static boolean isYoutubeMixId(final String playlistId) {
         return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
     }
 
     /**
-     * Checks if the given playlist id is a youtube music mix (auto-generated playlist)
+     * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
-     * Ids from a youtube music mix start with "RD"
+     * Ids from a YouTube Music Mix start with "RD"
      * @param playlistId
-     * @return Whether given id belongs to a youtube music mix
+     * @return Whether given id belongs to a YouTube Music Mix
      */
-    public static boolean isYoutubeMusicMixId(String playlistId) {
+    public static boolean isYoutubeMusicMixId(final String playlistId) {
         return playlistId.startsWith("RDAMVM");
     }
 
@@ -352,7 +362,7 @@ public class YoutubeParsingHelper {
                 .end()
                 .value("query", "test")
                 .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
-            .end().done().getBytes("UTF-8");
+            .end().done().getBytes(StandardCharsets.UTF_8);
         // @formatter:on
 
         Map<String, List<String>> headers = new HashMap<>();
@@ -436,10 +446,14 @@ public class YoutubeParsingHelper {
         } else if (navigationEndpoint.has("watchEndpoint")) {
             StringBuilder url = new StringBuilder();
             url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
-            if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
+            if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
-                url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
+                url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint")
-            if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
+                        .getString("playlistId"));
-                url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
+            }
+            if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
+                url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint")
+                        .getInt("startTimeSeconds"));
+            }
             return url.toString();
         } else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
             return "https://www.youtube.com/playlist?list=" +
@@ -467,7 +481,6 @@ public class YoutubeParsingHelper {
             if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
                 String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
                 if (!isNullOrEmpty(url)) {
-                    url = url.replaceAll("&", "&amp;");
                     textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
                     continue;
                 }
@@ -506,8 +519,8 @@ public class YoutubeParsingHelper {
     public static String getValidJsonResponseBody(final Response response)
             throws ParsingException, MalformedURLException {
         if (response.responseCode() == 404) {
-            throw new ContentNotAvailableException("Not found" +
+            throw new ContentNotAvailableException("Not found"
-                    " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
+                    + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
         }
 
         final String responseBody = response.responseBody();
@@ -527,13 +540,39 @@ public class YoutubeParsingHelper {
         final String responseContentType = response.getHeader("Content-Type");
         if (responseContentType != null
                 && responseContentType.toLowerCase().contains("text/html")) {
-            throw new ParsingException("Got HTML document, expected JSON response" +
+            throw new ParsingException("Got HTML document, expected JSON response"
-                    " (latest url was: \"" + response.latestUrl() + "\")");
+                    + " (latest url was: \"" + response.latestUrl() + "\")");
         }
 
         return responseBody;
     }
 
+    public static Response getResponse(final String url, final Localization localization)
+            throws IOException, ExtractionException {
+        final Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+
+        final Response response = getDownloader().get(url, headers, localization);
+        getValidJsonResponseBody(response);
+
+        return response;
+    }
+
+    public static String extractCookieValue(final String cookieName, final Response response) {
+        final List<String> cookies = response.responseHeaders().get("Set-Cookie");
+        int startIndex;
+        String result = "";
+        for (final String cookie : cookies) {
+            startIndex = cookie.indexOf(cookieName);
+            if (startIndex != -1) {
+                result = cookie.substring(startIndex + cookieName.length() + "=".length(),
+                        cookie.indexOf(";", startIndex));
+            }
+        }
+        return result;
+    }
+
     public static JsonArray getJsonResponse(final String url, final Localization localization)
             throws IOException, ExtractionException {
         Map<String, List<String>> headers = new HashMap<>();
@@ -541,8 +580,24 @@ public class YoutubeParsingHelper {
         headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
         final Response response = getDownloader().get(url, headers, localization);
 
-        final String responseBody = getValidJsonResponseBody(response);
+        return toJsonArray(getValidJsonResponseBody(response));
+    }
 
+    public static JsonArray getJsonResponse(final Page page, final Localization localization)
+            throws IOException, ExtractionException {
+        final Map<String, List<String>> headers = new HashMap<>();
+        if (!isNullOrEmpty(page.getCookies())) {
+            headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
+        }
+        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+
+        final Response response = getDownloader().get(page.getUrl(), headers, localization);
+
+        return toJsonArray(getValidJsonResponseBody(response));
+    }
+
+    public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
         try {
             return JsonParser.array().from(responseBody);
         } catch (JsonParserException e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -110,7 +110,7 @@ public class YoutubeService extends StreamingService {
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
+    public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
         if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
             return new YoutubeMixPlaylistExtractor(this, linkHandler);
         } else {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -4,8 +4,10 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -15,34 +17,50 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray;
 
 /**
- * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of
+ * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
- * "youtube.com/watch?v=videoId&list=playlistId"
+ * It handles URLs in the format of
+ * {@code youtube.com/watch?v=videoId&list=playlistId}
  */
 public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
 
+    /**
+     * YouTube identifies mixes based on this cookie. With this information it can generate
+     * continuations without duplicates.
+     */
+    private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE";
+
     private JsonObject initialData;
     private JsonObject playlistData;
+    private String cookieValue;
 
-    public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
+    public YoutubeMixPlaylistExtractor(final StreamingService service,
+                                       final ListLinkHandler linkHandler) {
         super(service, linkHandler);
     }
 
     @Override
-    public void onFetchPage(@Nonnull Downloader downloader)
+    public void onFetchPage(@Nonnull final Downloader downloader)
-        throws IOException, ExtractionException {
+            throws IOException, ExtractionException {
         final String url = getUrl() + "&pbj=1";
-        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
+        final Response response = getResponse(url, getExtractorLocalization());
+        final JsonArray ajaxJson = toJsonArray(response.responseBody());
         initialData = ajaxJson.getObject(3).getObject("response");
         playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
-            .getObject("playlist").getObject("playlist");
+                .getObject("playlist").getObject("playlist");
+        cookieValue = extractCookieValue(COOKIE_NAME, response);
     }
 
     @Nonnull
@@ -58,16 +76,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getThumbnailUrl() throws ParsingException {
         try {
-            final String playlistId = playlistData.getString("playlistId");
+            return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
+        } catch (final Exception e) {
             try {
-                return getThumbnailUrlFromPlaylistId(playlistId);
-            } catch (ParsingException e) {
                 //fallback to thumbnail of current video. Always the case for channel mix
                 return getThumbnailUrlFromVideoId(
                     initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
                         .getString("videoId"));
+            } catch (final Exception ignored) {
             }
-        } catch (Exception e) {
             throw new ParsingException("Could not get playlist thumbnail", e);
         }
     }
@@ -104,53 +121,56 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         collectStreamsFrom(collector, playlistData.getArray("contents"));
-        return new InfoItemsPage<>(collector, getNextPageUrl());
+        return new InfoItemsPage<>(collector,
+                new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue)));
     }
 
-    @Override
+    private String getNextPageUrl() throws ExtractionException {
-    public String getNextPageUrl() throws ExtractionException {
         return getNextPageUrlFrom(playlistData);
     }
 
-    private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException {
+    private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
-        final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
+        final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
-            .get(playlistData.getArray("contents").size() - 1));
+                .get(playlistJson.getArray("contents").size() - 1));
         if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
             throw new ExtractionException("Could not extract next page url");
         }
-        //Index of video in mix is missing, but adding it doesn't appear to have any effect.
+
-        //And since the index needs to be tracked by us, it is left out
         return getUrlFromNavigationEndpoint(
-            lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+                lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
-            + "&pbj=1";
+                + "&pbj=1";
     }
 
     @Override
-    public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
+    public InfoItemsPage<StreamInfoItem> getPage(final Page page)
             throws ExtractionException, IOException {
-        if (pageUrl == null || pageUrl.isEmpty()) {
+        if (page == null || page.getUrl().isEmpty()) {
             throw new ExtractionException(
                 new IllegalArgumentException("Page url is empty or null"));
         }
 
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());
-        final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
+        final JsonObject playlistJson =
-        JsonObject playlistData =
+                ajaxJson.getObject(3).getObject("response").getObject("contents")
-            ajaxJson.getObject(3).getObject("response").getObject("contents")
+                        .getObject("twoColumnWatchNextResults").getObject("playlist")
-                .getObject("twoColumnWatchNextResults").getObject("playlist")
+                        .getObject("playlist");
-                .getObject("playlist");
+        final JsonArray allStreams = playlistJson.getArray("contents");
-        final JsonArray streams = playlistData.getArray("contents");
+        // Sublist because youtube returns up to 24 previous streams in the mix
-        //Because continuation requests are created with the last video of previous request as start
+        // +1 because the stream of "currentIndex" was already extracted in previous request
-        streams.remove(0);
+        final List<Object> newStreams =
-        collectStreamsFrom(collector, streams);
+                allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
-        return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData));
+
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        collectStreamsFrom(collector, newStreams);
+        return new InfoItemsPage<>(collector,
+                new Page(getNextPageUrlFrom(playlistJson), page.getCookies()));
     }
 
     private void collectStreamsFrom(
-            @Nonnull StreamInfoItemsCollector collector,
+            @Nonnull final StreamInfoItemsCollector collector,
-            @Nullable JsonArray streams) {
+            @Nullable final List<Object> streams) {
 
         if (streams == null) {
             return;
@@ -158,9 +178,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
 
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        for (Object stream : streams) {
+        for (final Object stream : streams) {
             if (stream instanceof JsonObject) {
-                JsonObject streamInfo = ((JsonObject) stream)
+                final JsonObject streamInfo = ((JsonObject) stream)
                     .getObject("playlistPanelVideoRenderer");
                 if (streamInfo != null) {
                     collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
@@ -169,7 +189,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         }
     }
 
-    private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException {
+    private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException {
         final String videoId;
         if (playlistId.startsWith("RDMM")) {
             videoId = playlistId.substring(4);
@@ -184,7 +204,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         return getThumbnailUrlFromVideoId(videoId);
     }
 
-    private String getThumbnailUrlFromVideoId(String videoId) {
+    private String getThumbnailUrlFromVideoId(final String videoId) {
         return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java

@@ -14,82 +14,88 @@ import java.util.List;
 
 public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
 
-    private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory();
+    private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
+            new YoutubePlaylistLinkHandlerFactory();
 
     public static YoutubePlaylistLinkHandlerFactory getInstance() {
-        return instance;
+        return INSTANCE;
     }
 
     @Override
-    public String getUrl(String id, List<String> contentFilters, String sortFilter) {
+    public String getUrl(final String id, final List<String> contentFilters,
+                         final String sortFilter) {
         return "https://www.youtube.com/playlist?list=" + id;
     }
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public String getId(final String url) throws ParsingException {
         try {
-            URL urlObj = Utils.stringToURL(url);
+            final URL urlObj = Utils.stringToURL(url);
 
             if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
                     || YoutubeParsingHelper.isInvidioURL(urlObj))) {
                 throw new ParsingException("the url given is not a Youtube-URL");
             }
 
-            String path = urlObj.getPath();
+            final String path = urlObj.getPath();
             if (!path.equals("/watch") && !path.equals("/playlist")) {
                 throw new ParsingException("the url given is neither a video nor a playlist URL");
             }
 
-            String listID = Utils.getQueryValue(urlObj, "list");
+            final String listID = Utils.getQueryValue(urlObj, "list");
 
             if (listID == null) {
                 throw new ParsingException("the url given does not include a playlist");
             }
 
             if (!listID.matches("[a-zA-Z0-9_-]{10,}")) {
-                throw new ParsingException("the list-ID given in the URL does not match the list pattern");
+                throw new ParsingException(
+                        "the list-ID given in the URL does not match the list pattern");
             }
 
-            // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists
+            if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) {
-            if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) {
+                throw new ContentNotSupportedException(
-                throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported");
+                        "YouTube Music Mix playlists are not yet supported");
             }
 
             return listID;
         } catch (final Exception exception) {
-            throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception);
+            throw new ParsingException("Error could not parse url :" + exception.getMessage(),
+                    exception);
         }
     }
 
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
-            String playlistId = getId(url);
+            getId(url);
-            //Because youtube music mix are not supported yet.
-            return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId);
         } catch (ParsingException e) {
             return false;
         }
+        return true;
     }
 
     /**
-     * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like
+     * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is
-     * youtube.com/watch?v=videoId&list=playlistId
+     * like
+     * <code>https://youtube.com/watch?v=videoId&list=playlistId</code>.
      * <p>Otherwise use super</p>
      */
     @Override
-    public ListLinkHandler fromUrl(String url) throws ParsingException {
+    public ListLinkHandler fromUrl(final String url) throws ParsingException {
         try {
-            URL urlObj = Utils.stringToURL(url);
+            final URL urlObj = Utils.stringToURL(url);
-            String listID = Utils.getQueryValue(urlObj, "list");
+            final String listID = Utils.getQueryValue(urlObj, "list");
             if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) {
                 String videoID = Utils.getQueryValue(urlObj, "v");
                 if (videoID == null) {
                     videoID = listID.substring(2);
                 }
-                String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID;
+                final String newUrl = "https://www.youtube.com/watch?v=" + videoID
-                return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url),
+                    + "&list=" + listID;
-                    getSortFilter(url));
+                return new ListLinkHandler(new LinkHandler(url, newUrl, listID),
+                        getContentFilter(url),
+                        getSortFilter(url));
             }
         } catch (MalformedURLException exception) {
             throw new ParsingException("Error could not parse url :" + exception.getMessage(),

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -8,6 +8,7 @@ import java.net.URL;
 import java.net.URLDecoder;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -260,4 +261,15 @@ public class Utils {
         }
         return stringBuilder.toString();
     }
+
+    public static String join(final String delimiter, final String mapJoin,
+        final Map<? extends CharSequence, ? extends CharSequence> elements) {
+        final List<String> list = new LinkedList<>();
+        for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
+            .entrySet()) {
+            list.add(entry.getKey() + mapJoin + entry.getValue());
+        }
+        return join(delimiter, list);
+    }
+
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -2,26 +2,45 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.startsWith;
-import static org.junit.Assert.*;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
+import java.util.HashSet;
+import java.util.Set;
+import org.hamcrest.MatcherAssert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+@RunWith(Suite.class)
+@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
 public class YoutubeMixPlaylistExtractorTest {
 
+    public static final String PBJ = "&pbj=1";
+    private static final String VIDEO_ID = "_AzeUSL9lZc";
+    private static final String VIDEO_TITLE =
+            "Most Beautiful And Emotional  Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
+
     private static YoutubeMixPlaylistExtractor extractor;
-    private static String videoId = "_AzeUSL9lZc";
-    private static String videoTitle = "Most Beautiful And Emotional  Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
 
     public static class Mix {
 
@@ -29,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
+                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
             extractor.fetchPage();
         }
 
@@ -41,81 +60,82 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getName() throws Exception {
-            String name = extractor.getName();
+            final String name = extractor.getName();
             assertThat(name, startsWith("Mix"));
-            assertThat(name, containsString(videoTitle));
+            assertThat(name, containsString(VIDEO_TITLE));
         }
 
         @Test
         public void getThumbnailUrl() throws Exception {
             final String thumbnailUrl = extractor.getThumbnailUrl();
             assertIsSecureUrl(thumbnailUrl);
-            assertThat(thumbnailUrl, containsString("yt"));
+            MatcherAssert.assertThat(thumbnailUrl, containsString("yt"));
-            assertThat(thumbnailUrl, containsString(videoId));
+            assertThat(thumbnailUrl, containsString(VIDEO_ID));
-        }
-
-        @Test
-        public void getNextPageUrl() throws Exception {
-            final String nextPageUrl = extractor.getNextPageUrl();
-            assertIsSecureUrl(nextPageUrl);
-            assertThat(nextPageUrl, containsString("list=RD" + videoId));
         }
 
         @Test
         public void getInitialPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
         public void getPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+                new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
+                    + PBJ));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
-        public void getPageMultipleTimes() throws Exception {
+        public void getContinuations() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final Set<String> urls = new HashSet<>();
 
             //Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
-                streams = extractor.getPage(streams.getNextPageUrl());
+                for (final StreamInfoItem item : streams.getItems()) {
+                    assertFalse(urls.contains(item.getUrl()));
+                    urls.add(item.getUrl());
+                }
+
+                streams = extractor.getPage(streams.getNextPage());
             }
             assertTrue(streams.hasNextPage());
             assertFalse(streams.getItems().isEmpty());
         }
 
         @Test
-        public void getStreamCount() throws Exception {
+        public void getStreamCount() {
             assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
         }
     }
 
     public static class MixWithIndex {
 
-        private static String index = "&index=13";
+        private static final String INDEX = "&index=13";
-        private static String videoIdNumber13 = "qHtzO49SDmk";
+        private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk";
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                 .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId
+                    "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
-                        + index);
+                        + VIDEO_ID + INDEX);
             extractor.fetchPage();
         }
 
         @Test
         public void getName() throws Exception {
-            String name = extractor.getName();
+            final String name = extractor.getName();
             assertThat(name, startsWith("Mix"));
-            assertThat(name, containsString(videoTitle));
+            assertThat(name, containsString(VIDEO_TITLE));
         }
 
         @Test
@@ -123,47 +143,47 @@ public class YoutubeMixPlaylistExtractorTest {
             final String thumbnailUrl = extractor.getThumbnailUrl();
             assertIsSecureUrl(thumbnailUrl);
             assertThat(thumbnailUrl, containsString("yt"));
-            assertThat(thumbnailUrl, containsString(videoId));
+            assertThat(thumbnailUrl, containsString(VIDEO_ID));
-        }
-
-        @Test
-        public void getNextPageUrl() throws Exception {
-            final String nextPageUrl = extractor.getNextPageUrl();
-            assertIsSecureUrl(nextPageUrl);
-            assertThat(nextPageUrl, containsString("list=RD" + videoId));
         }
 
         @Test
         public void getInitialPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
         public void getPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+                new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+                    + VIDEO_ID + INDEX + PBJ));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
-        public void getPageMultipleTimes() throws Exception {
+        public void getContinuations() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final Set<String> urls = new HashSet<>();
 
             //Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
+                for (final StreamInfoItem item : streams.getItems()) {
+                    assertFalse(urls.contains(item.getUrl()));
+                    urls.add(item.getUrl());
+                }
 
-                streams = extractor.getPage(streams.getNextPageUrl());
+                streams = extractor.getPage(streams.getNextPage());
             }
             assertTrue(streams.hasNextPage());
             assertFalse(streams.getItems().isEmpty());
         }
 
         @Test
-        public void getStreamCount() throws Exception {
+        public void getStreamCount() {
             assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
         }
     }
@@ -175,7 +195,8 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                 .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId);
+                    "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+                        + VIDEO_ID);
             extractor.fetchPage();
         }
 
@@ -186,7 +207,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getName() throws Exception {
-            String name = extractor.getName();
+            final String name = extractor.getName();
             assertEquals("My Mix", name);
         }
 
@@ -197,44 +218,44 @@ public class YoutubeMixPlaylistExtractorTest {
             assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
         }
 
-        @Test
-        public void getNextPageUrl() throws Exception {
-            final String nextPageUrl = extractor.getNextPageUrl();
-            assertIsSecureUrl(nextPageUrl);
-            assertThat(nextPageUrl, containsString("list=RDMM" + videoId));
-        }
-
         @Test
         public void getInitialPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
         public void getPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            final InfoItemsPage<StreamInfoItem> streams =
+                extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+                    + "&list=RDMM" + VIDEO_ID + PBJ));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
-
         @Test
-        public void getPageMultipleTimes() throws Exception {
+        public void getContinuations() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final Set<String> urls = new HashSet<>();
 
             //Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
-                streams = extractor.getPage(streams.getNextPageUrl());
+                for (final StreamInfoItem item : streams.getItems()) {
+                    assertFalse(urls.contains(item.getUrl()));
+                    urls.add(item.getUrl());
+                }
+
+                streams = extractor.getPage(streams.getNextPage());
             }
             assertTrue(streams.hasNextPage());
             assertFalse(streams.getItems().isEmpty());
         }
 
         @Test
-        public void getStreamCount() throws Exception {
+        public void getStreamCount() {
             assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
         }
     }
@@ -249,10 +270,10 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test(expected = ExtractionException.class)
         public void getPageEmptyUrl() throws Exception {
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                .getPlaylistExtractor(
+                    .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
+                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
             extractor.fetchPage();
-            extractor.getPage("");
+            extractor.getPage(new Page(""));
         }
 
         @Test(expected = ExtractionException.class)
@@ -267,10 +288,9 @@ public class YoutubeMixPlaylistExtractorTest {
 
     public static class ChannelMix {
 
-        private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw";
+        private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw";
-        private static String videoIdOfChannel = "mnk6gnOBYIo";
+        private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo";
-        private static String channelTitle = "Linus Tech Tips";
+        private static final String CHANNEL_TITLE = "Linus Tech Tips";
-
 
 
         @BeforeClass
@@ -278,15 +298,16 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                 .getPlaylistExtractor(
-                    "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId);
+                    "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                        + "&list=RDCM" + CHANNEL_ID);
             extractor.fetchPage();
         }
 
         @Test
         public void getName() throws Exception {
-            String name = extractor.getName();
+            final String name = extractor.getName();
             assertThat(name, startsWith("Mix"));
-            assertThat(name, containsString(channelTitle));
+            assertThat(name, containsString(CHANNEL_TITLE));
         }
 
         @Test
@@ -296,30 +317,25 @@ public class YoutubeMixPlaylistExtractorTest {
             assertThat(thumbnailUrl, containsString("yt"));
         }
 
-        @Test
-        public void getNextPageUrl() throws Exception {
-            final String nextPageUrl = extractor.getNextPageUrl();
-            assertIsSecureUrl(nextPageUrl);
-            assertThat(nextPageUrl, containsString("list=RDCM" + channelId));
-        }
-
         @Test
         public void getInitialPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
         public void getPage() throws Exception {
-            InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+                new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                    + "&list=RDCM" + CHANNEL_ID + PBJ));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
 
         @Test
-        public void getStreamCount() throws Exception {
+        public void getStreamCount() {
             assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
         }
     }
-}
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java

@@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
         assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist
-        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java

@@ -62,14 +62,14 @@ public class YoutubeServiceTest {
 
     @Test
     public void getPlayListExtractorIsNormalPlaylist() throws Exception {
-        PlaylistExtractor extractor = service.getPlaylistExtractor(
+        final PlaylistExtractor extractor = service.getPlaylistExtractor(
             "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS");
         assertTrue(extractor instanceof YoutubePlaylistExtractor);
     }
 
     @Test
     public void getPlaylistExtractorIsMix() throws Exception {
-        String videoId = "_AzeUSL9lZc";
+        final String videoId = "_AzeUSL9lZc";
         PlaylistExtractor extractor = YouTube.getPlaylistExtractor(
             "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
         assertTrue(extractor instanceof YoutubeMixPlaylistExtractor);