From ecfc3706858b0b8da4a0284a02b85556f00e46c8 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sat, 30 Jul 2022 16:05:52 +0200 Subject: [PATCH] Fixed all YTMixPlaylists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳 --- .../exceptions/ConsentRequiredException.java | 12 +++++ .../youtube/YoutubeParsingHelper.java | 45 +++++++++------- .../YoutubeMixPlaylistExtractor.java | 52 +++++++++++-------- .../YoutubeMixPlaylistExtractorTest.java | 30 +++++++---- .../services/youtube/YoutubeTestsUtils.java | 1 + 5 files changed, 89 insertions(+), 51 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java new file mode 100644 index 000000000..90feacc0b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.exceptions; + +public class ConsentRequiredException extends ParsingException { + + public ConsentRequiredException(final String message) { + super(message); + } + + public ConsentRequiredException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 2458a97cd..052316285 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -27,7 +27,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - import static java.util.Collections.singletonList; import com.grack.nanojson.JsonArray; @@ -245,16 +244,19 @@ public final class YoutubeParsingHelper { * The three digits at the end can be random, but are required. *

*/ - private static final String CONSENT_COOKIE_VALUE = "PENDING+"; - + private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+"; /** - * YouTube {@code CONSENT} cookie. + * {@code YES+} means that the user did submit their choices and accepted all cookies. * *

- * Should prevent redirect to {@code consent.youtube.com}. + * Therefore, YouTube & Google can track the user, because they did give consent. + *

+ * + *

+ * The three digits at the end can be random, but are required. *

*/ - private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE; + private static final String CONSENT_COOKIE_YES_VALUE = "YES+"; private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; @@ -265,6 +267,13 @@ public final class YoutubeParsingHelper { private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); + /** + * {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}. + *
+ * {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}. + */ + private static boolean consentAccepted = false; + private static boolean isGoogleURL(final String url) { final String cachedUrl = extractCachedUrlIfNeeded(url); try { @@ -1378,7 +1387,6 @@ public final class YoutubeParsingHelper { /** * Add the CONSENT cookie to prevent redirect to consent.youtube.com - * @see #CONSENT_COOKIE * @param headers the headers which should be completed */ public static void addCookieHeader(@Nonnull final Map> headers) { @@ -1391,8 +1399,9 @@ public final class YoutubeParsingHelper { @Nonnull public static String generateConsentCookie() { - final int statusCode = 100 + numberGenerator.nextInt(900); - return CONSENT_COOKIE + statusCode; + return "CONSENT=" + + (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE) + + (100 + numberGenerator.nextInt(900)); } public static String extractCookieValue(final String cookieName, @@ -1612,16 +1621,6 @@ public final class YoutubeParsingHelper { return false; } - @Nonnull - public static String unescapeDocument(@Nonnull final String doc) { - return doc - .replaceAll("\\\\x22", "\"") - .replaceAll("\\\\x7b", "{") - .replaceAll("\\\\x7d", "}") - .replaceAll("\\\\x5b", "[") - .replaceAll("\\\\x5d", "]"); - } - /** * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in * playback requests (and also for some clients, in the player request body). @@ -1692,4 +1691,12 @@ public final class YoutubeParsingHelper { public static boolean isIosStreamingUrl(@Nonnull final String url) { return Parser.isMatch(C_IOS_PATTERN, url); } + + public static void setConsentAccepted(final boolean accepted) { + consentAccepted = accepted; + } + + public static boolean isConsentAccepted() { + return consentAccepted; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 49225330b..dc7179e47 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,28 +1,15 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static org.schabi.newpipe.extractor.utils.Utils.stringToURL; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -35,6 +22,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -43,8 +32,18 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.stringToURL; /** * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). @@ -89,16 +88,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8); final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + // Cookie is required due to consent + addYouTubeHeaders(headers); final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); - playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("playlist").getObject("playlist"); + playlistData = initialData + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("playlist") + .getObject("playlist"); if (isNullOrEmpty(playlistData)) { - throw new ExtractionException("Could not get playlistData"); + final ExtractionException ex = new ExtractionException("Could not get playlistData"); + if (!YoutubeParsingHelper.isConsentAccepted()) { + throw new ConsentRequiredException( + "Consent is required in some countries to view Mix playlists", + ex); + } + throw ex; } cookieValue = extractCookieValue(COOKIE_NAME, response); } @@ -212,7 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + // Cookie is required due to consent + addYouTubeHeaders(headers); final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index b562d6594..899067700 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -1,15 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; - import com.grack.nanojson.JsonWriter; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderFactory; @@ -31,6 +22,17 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; + public class YoutubeMixPlaylistExtractorTest { private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/"; @@ -45,6 +47,7 @@ public class YoutubeMixPlaylistExtractorTest { @BeforeAll public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube @@ -140,6 +143,7 @@ public class YoutubeMixPlaylistExtractorTest { @BeforeAll public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube @@ -221,11 +225,12 @@ public class YoutubeMixPlaylistExtractorTest { } public static class MyMix { - private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_ID = "YVkUvmDQ3HY"; @BeforeAll public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube @@ -249,7 +254,7 @@ public class YoutubeMixPlaylistExtractorTest { void getThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); - assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); + assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID)); } @Test @@ -316,6 +321,7 @@ public class YoutubeMixPlaylistExtractorTest { @BeforeAll public static void setUp() throws IOException { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); } @@ -350,6 +356,7 @@ public class YoutubeMixPlaylistExtractorTest { @BeforeAll public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube @@ -414,6 +421,7 @@ public class YoutubeMixPlaylistExtractorTest { @BeforeAll public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); + YoutubeParsingHelper.setConsentAccepted(true); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index cc2b3111e..a98039873 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -21,6 +21,7 @@ public final class YoutubeTestsUtils { *

*/ public static void ensureStateless() { + YoutubeParsingHelper.setConsentAccepted(false); YoutubeParsingHelper.resetClientVersionAndKey(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); YoutubeStreamExtractor.resetDeobfuscationCode();