Fixed all YTMixPlaylists

Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
2022-07-30 16:05:52 +02:00 · 2022-07-30 16:05:52 +02:00 · ecfc370685
parent 2b6fe294b2
commit ecfc370685
5 changed files with 89 additions and 51 deletions
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java
@ -0,0 +1,12 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+public class ConsentRequiredException extends ParsingException {
+
+    public ConsentRequiredException(final String message) {
+        super(message);
+    }
+
+    public ConsentRequiredException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+}
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@ -27,7 +27,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
 import static java.util.Collections.singletonList;

 import com.grack.nanojson.JsonArray;
@ -245,16 +244,19 @@ public final class YoutubeParsingHelper {
     * The three digits at the end can be random, but are required.
     * </p>
     */
-    private static final String CONSENT_COOKIE_VALUE = "PENDING+";
-
+    private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
    /**
-     * YouTube {@code CONSENT} cookie.
+     * {@code YES+} means that the user did submit their choices and accepted all cookies.
     *
     * <p>
-     * Should prevent redirect to {@code consent.youtube.com}.
+     * Therefore, YouTube & Google can track the user, because they did give consent.
+     * </p>
+     *
+     * <p>
+     * The three digits at the end can be random, but are required.
     * </p>
     */
-    private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
+    private static final String CONSENT_COOKIE_YES_VALUE = "YES+";

    private static final String FEED_BASE_CHANNEL_ID =
            "https://www.youtube.com/feeds/videos.xml?channel_id=";
@ -265,6 +267,13 @@ public final class YoutubeParsingHelper {
    private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
    private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");

+    /**
+     * {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
+     * <br/>
+     * {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
+     */
+    private static boolean consentAccepted = false;
+
    private static boolean isGoogleURL(final String url) {
        final String cachedUrl = extractCachedUrlIfNeeded(url);
        try {
@ -1378,7 +1387,6 @@ public final class YoutubeParsingHelper {

    /**
     * Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
-     * @see #CONSENT_COOKIE
     * @param headers the headers which should be completed
     */
    public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@ -1391,8 +1399,9 @@ public final class YoutubeParsingHelper {

    @Nonnull
    public static String generateConsentCookie() {
-        final int statusCode = 100 + numberGenerator.nextInt(900);
-        return CONSENT_COOKIE + statusCode;
+        return "CONSENT="
+                + (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
+                + (100 + numberGenerator.nextInt(900));
    }

    public static String extractCookieValue(final String cookieName,
@ -1612,16 +1621,6 @@ public final class YoutubeParsingHelper {
        return false;
    }

-    @Nonnull
-    public static String unescapeDocument(@Nonnull final String doc) {
-        return doc
-                .replaceAll("\\\\x22", "\"")
-                .replaceAll("\\\\x7b", "{")
-                .replaceAll("\\\\x7d", "}")
-                .replaceAll("\\\\x5b", "[")
-                .replaceAll("\\\\x5d", "]");
-    }
-
    /**
     * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
     * playback requests (and also for some clients, in the player request body).
@ -1692,4 +1691,12 @@ public final class YoutubeParsingHelper {
    public static boolean isIosStreamingUrl(@Nonnull final String url) {
        return Parser.isMatch(C_IOS_PATTERN, url);
    }
+
+    public static void setConsentAccepted(final boolean accepted) {
+        consentAccepted = accepted;
+    }
+
+    public static boolean isConsentAccepted() {
+        return consentAccepted;
+    }
 }
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
@ -1,28 +1,15 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;

-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
-import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
-
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonBuilder;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
-
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -35,6 +22,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;

+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@ -43,8 +32,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;

-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;

 /**
 * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@ -89,16 +88,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
        final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);

        final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
+        // Cookie is required due to consent
+        addYouTubeHeaders(headers);

        final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
                + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);

        initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
-        playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
-                .getObject("playlist").getObject("playlist");
+        playlistData = initialData
+                .getObject("contents")
+                .getObject("twoColumnWatchNextResults")
+                .getObject("playlist")
+                .getObject("playlist");
        if (isNullOrEmpty(playlistData)) {
-            throw new ExtractionException("Could not get playlistData");
+            final ExtractionException ex = new ExtractionException("Could not get playlistData");
+            if (!YoutubeParsingHelper.isConsentAccepted()) {
+                throw new ConsentRequiredException(
+                        "Consent is required in some countries to view Mix playlists",
+                        ex);
+            }
+            throw ex;
        }
        cookieValue = extractCookieValue(COOKIE_NAME, response);
    }
@ -212,7 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {

        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
        final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
+        // Cookie is required due to consent
+        addYouTubeHeaders(headers);

        final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
                getExtractorLocalization());
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
@ -1,15 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube;

-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-
 import com.grack.nanojson.JsonWriter;
-
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@ -31,6 +22,17 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;

+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+
 public class YoutubeMixPlaylistExtractorTest {

    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
@ -45,6 +47,7 @@ public class YoutubeMixPlaylistExtractorTest {
        @BeforeAll
        public static void setUp() throws Exception {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
            extractor = (YoutubeMixPlaylistExtractor) YouTube
@ -140,6 +143,7 @@ public class YoutubeMixPlaylistExtractorTest {
        @BeforeAll
        public static void setUp() throws Exception {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
            extractor = (YoutubeMixPlaylistExtractor) YouTube
@ -221,11 +225,12 @@ public class YoutubeMixPlaylistExtractorTest {
    }

    public static class MyMix {
-        private static final String VIDEO_ID = "_AzeUSL9lZc";
+        private static final String VIDEO_ID = "YVkUvmDQ3HY";

        @BeforeAll
        public static void setUp() throws Exception {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
            extractor = (YoutubeMixPlaylistExtractor) YouTube
@ -249,7 +254,7 @@ public class YoutubeMixPlaylistExtractorTest {
        void getThumbnailUrl() throws Exception {
            final String thumbnailUrl = extractor.getThumbnailUrl();
            assertIsSecureUrl(thumbnailUrl);
-            assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
+            assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
        }

        @Test
@ -316,6 +321,7 @@ public class YoutubeMixPlaylistExtractorTest {
        @BeforeAll
        public static void setUp() throws IOException {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
        }
@ -350,6 +356,7 @@ public class YoutubeMixPlaylistExtractorTest {
        @BeforeAll
        public static void setUp() throws Exception {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
            extractor = (YoutubeMixPlaylistExtractor) YouTube
@ -414,6 +421,7 @@ public class YoutubeMixPlaylistExtractorTest {
        @BeforeAll
        public static void setUp() throws Exception {
            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
            extractor = (YoutubeMixPlaylistExtractor) YouTube
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
@ -21,6 +21,7 @@ public final class YoutubeTestsUtils {
     * </p>
     */
    public static void ensureStateless() {
+        YoutubeParsingHelper.setConsentAccepted(false);
        YoutubeParsingHelper.resetClientVersionAndKey();
        YoutubeParsingHelper.setNumberGenerator(new Random(1));
        YoutubeStreamExtractor.resetDeobfuscationCode();