[YouTube] Set CONSENT cookie

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -28,12 +28,7 @@ import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeParseException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -79,6 +74,17 @@ public class YoutubeParsingHelper {
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
     private static String[] youtubeMusicKeys;
 
+    /**
+     * <code>PENDING+</code> means that the user did not yet submit their choices.
+     * Therefore, YouTube & Google should not track the user, because they did not give consent.
+     * The three digits at the end can be random, but are required.
+     */
+    public static final String CONSENT_COOKIE_VALUE = "PENDING+" + (100 + new Random().nextInt(900));
+    /**
+     * Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
+     */
+    public static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
+
     private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
 
@@ -427,6 +433,7 @@ public class YoutubeParsingHelper {
         headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
         headers.put("Referer", Collections.singletonList("music.youtube.com"));
         headers.put("Content-Type", Collections.singletonList("application/json"));
+        addCookieHeader(headers);
 
         final String response = getDownloader().post(url, headers, json).responseBody();
 
@@ -629,8 +636,7 @@ public class YoutubeParsingHelper {
     public static Response getResponse(final String url, final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        addYouTubeHeaders(headers);
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
 
         final Response response = getDownloader().get(url, headers, localization);
         getValidJsonResponseBody(response);
@@ -638,6 +644,64 @@ public class YoutubeParsingHelper {
         return response;
     }
 
+    public static JsonArray getJsonResponse(final String url, final Localization localization)
+            throws IOException, ExtractionException {
+        Map<String, List<String>> headers = new HashMap<>();
+        addYouTubeHeaders(headers);
+
+        final Response response = getDownloader().get(url, headers, localization);
+
+        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
+    }
+
+    public static JsonArray getJsonResponse(final Page page, final Localization localization)
+            throws IOException, ExtractionException {
+        final Map<String, List<String>> headers = new HashMap<>();
+        addYouTubeHeaders(headers);
+
+        final Response response = getDownloader().get(page.getUrl(), headers, localization);
+
+        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
+    }
+
+    /**
+     * Add required headers and cookies to an existing headers Map.
+     * @see #addClientInfoHeaders(Map)
+     * @see #addCookieHeader(Map)
+     */
+    public static void addYouTubeHeaders(final Map<String, List<String>> headers)
+            throws IOException, ExtractionException {
+        addClientInfoHeaders(headers);
+        addCookieHeader(headers);
+    }
+
+    /**
+     * Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
+     * @param headers The headers which should be completed
+     */
+    public static void addClientInfoHeaders(final Map<String, List<String>> headers)
+            throws IOException, ExtractionException {
+        if (headers.get("X-YouTube-Client-Name") == null) {
+            headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        }
+        if (headers.get("X-YouTube-Client-Version") == null) {
+            headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        }
+    }
+
+    /**
+     * Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
+     * @see #CONSENT_COOKIE
+     * @param headers the headers which should be completed
+     */
+    public static void addCookieHeader(final Map<String, List<String>> headers) {
+        if (headers.get("Cookie") == null) {
+            headers.put("Cookie", Arrays.asList(CONSENT_COOKIE));
+        } else {
+            headers.get("Cookie").add(CONSENT_COOKIE);
+        }
+    }
+
     public static String extractCookieValue(final String cookieName, final Response response) {
         final List<String> cookies = response.responseHeaders().get("set-cookie");
         int startIndex;
@@ -652,30 +716,6 @@ public class YoutubeParsingHelper {
         return result;
     }
 
-    public static JsonArray getJsonResponse(final String url, final Localization localization)
-            throws IOException, ExtractionException {
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
-        final Response response = getDownloader().get(url, headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
-
-    public static JsonArray getJsonResponse(final Page page, final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        if (!isNullOrEmpty(page.getCookies())) {
-            headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
-        }
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
-
-        final Response response = getDownloader().get(page.getUrl(), headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
-
     /**
      * Shared alert detection function, multiple endpoints return the error similarly structured.
      * <p>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -20,7 +20,9 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -130,8 +132,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         collectStreamsFrom(collector, playlistData.getArray("contents"));
-        return new InfoItemsPage<>(collector,
+
-                new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
+        final Map<String, String> cookies = new HashMap<>();
+        cookies.put(COOKIE_NAME, cookieValue);
+
+        return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
     }
 
     private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -12,9 +12,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
 import java.io.IOException;
 import java.net.URLEncoder;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONSENT_COOKIE_VALUE;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
 /*
@@ -45,17 +46,20 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
 
     @Override
     public List<String> suggestionList(String query) throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
+        final Downloader dl = NewPipe.getDownloader();
-        List<String> suggestions = new ArrayList<>();
+        final List<String> suggestions = new ArrayList<>();
 
-        String url = "https://suggestqueries.google.com/complete/search"
+        final String url = "https://suggestqueries.google.com/complete/search"
                 + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
                 + "&jsonp=" + "JP"
                 + "&ds=" + "yt"
                 + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
                 + "&q=" + URLEncoder.encode(query, UTF_8);
 
-        String response = dl.get(url, getExtractorLocalization()).responseBody();
+        final Map<String, List<String>> headers = new HashMap<>();
+        addCookieHeader(headers);
+
+        String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
         // trim JSONP part "JP(...)"
         response = response.substring(3, response.length() - 1);
         try {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -21,10 +21,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlayli
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.*;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.startsWith;
@@ -44,8 +41,7 @@ public class YoutubeMixPlaylistExtractorTest {
     private static final String VIDEO_TITLE =
             "Most Beautiful And Emotional  Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
-    private static final Map<String, String> dummyCookie
+    private static final Map<String, String> dummyCookie = new HashMap<>();
-            = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
 
     private static YoutubeMixPlaylistExtractor extractor;
 
@@ -55,6 +51,7 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
+            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor(
                             "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
@@ -133,6 +130,7 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
+            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor(
                             "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
@@ -203,6 +201,7 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
+            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor(
                             "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
@@ -277,6 +276,7 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws IOException {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "invalid"));
+            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
         }
 
         @Test(expected = IllegalArgumentException.class)
@@ -309,6 +309,7 @@ public class YoutubeMixPlaylistExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
+            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor(
                             "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL