Use the youtubei.googleapis.com domain for YouTube Music searches + change a check + update client version and mocks

Change the domain from music.youtube.com to youtubei.googleapis.com.
Use a lightweight request to check if the hardcoded YouTubeMusic keys are valid. Increase the length of the response to 500 because if the key is invalid, the length of the response returned is higher than 250 and the response when the key is valid is higher than 1500.
Format the YoutubeMusicSearchExtractor file.
Update YouTube web client version and mocks

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -64,7 +64,7 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
-    private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20210408.08.00";
     private static String clientVersion;
 
     private static String key;
@@ -413,7 +413,8 @@ public class YoutubeParsingHelper {
     }
 
     public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
-        final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
+        final String url = "https://youtubei.googleapis.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
+                + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
 
         // @formatter:off
         byte[] json = JsonWriter.string()
@@ -440,8 +441,7 @@ public class YoutubeParsingHelper {
                         .value("enableSafetyMode", false)
                     .end()
                 .end()
-                .value("query", "test")
-                .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
+                .value("input", "")
             .end().done().getBytes(UTF_8);
         // @formatter:on
 
@@ -455,7 +455,7 @@ public class YoutubeParsingHelper {
 
         final String response = getDownloader().post(url, headers, json).responseBody();
 
-        return response.length() > 50; // ensure to have a valid response
+        return response.length() > 500; // ensure to have a valid response
     }
 
     public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -667,4 +667,3 @@ public class YoutubeChannelExtractorTest {
         }
     }
 }
-

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java

@@ -17,7 +17,7 @@ import javax.annotation.Nullable;
 import static java.util.Collections.singletonList;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
-// Doesn't work with mocks. Makes request with different `dataToSend` i think
+// Doesn't work with mocks. Makes request with different `dataToSend` I think
 public class YoutubeMusicSearchExtractorTest {
     public static class MusicSongs extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java

@@ -278,7 +278,6 @@ public class YoutubeSearchExtractorTest {
         @Override public String expectedId() { return QUERY; }
         @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
         @Override public String expectedOriginalUrlContains() throws Exception { return "youtube.com/results?search_query=" + QUERY; }
-
     }
 
     public static class ChannelVerified extends DefaultSearchExtractorTest {
@@ -318,5 +317,4 @@ public class YoutubeSearchExtractorTest {
             assertTrue(verified);
         }
     }
-
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -21,6 +21,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 /**
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
+
 @Ignore("Video is not available in specific countries. Someone else has to generate mocks")
 public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
@@ -59,5 +60,4 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @Override public List<String> expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); }
     @Override public String expectedCategory() { return "Entertainment"; }
     @Override public String expectedLicence() { return "YouTube licence"; }
-
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_9.json

@@ -150,7 +150,7 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Tue, 08 Jun 2021 13:14:16 GMT"
+        "Tue, 08 Jun 2021 13:18:50 GMT"
       ],
       "server": [
         "ESF"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/invalid/generated_mock_1.json

@@ -7,13 +7,13 @@
         "en-GB, en;q\u003d0.9"
       ],
       "Cookie": [
-        "CONSENT\u003dPENDING+385"
+        "CONSENT\u003dPENDING+100285"
       ],
       "X-YouTube-Client-Name": [
         "1"
       ],
       "X-YouTube-Client-Version": [
-        "2.20200214.04.00"
+        "2.20210408.08.00"
       ]
     },
     "localization": {
@@ -23,10 +23,10 @@
   },
   "response": {
     "responseCode": 200,
-    "responseMessage": "",
+    "responseMessage": "OK",
     "responseHeaders": {
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "no-cache, no-store, max-age\u003d0, must-revalidate"
@@ -38,7 +38,7 @@
         "application/json; charset\u003dutf-8"
       ],
       "date": [
-        "Sat, 03 Jul 2021 11:29:31 GMT"
+        "Sun, 11 Apr 2021 15:57:14 GMT"
       ],
       "expires": [
         "Mon, 01 Jan 1990 00:00:00 GMT"
@@ -46,9 +46,6 @@
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
       ],
-      "permissions-policy": [
-        "ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
-      ],
       "pragma": [
         "no-cache"
       ],
@@ -56,11 +53,15 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d6I04qC_jDQY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "YSC\u003ddhXDf1Ev5bE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+790; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
       ],
+      "transfer-encoding": [
+        "chunked"
+      ],
       "x-content-type-options": [
         "nosniff"
       ],
@@ -74,7 +75,7 @@
         "0"
       ]
     },
-    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r4---sn-4g5ednld.googlevideo.com\\/generate_204\",\"https:\\/\\/r4---sn-4g5ednld.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23968386,23891346,23885487,23966208,24058240,23975059,23884386,24012512,24056839,24023960,23804281,24030040,24045412,23999405,24045470,23821390,24049575,24060173,23946420,24037794,24042870,24003103,24059009,1714254,24007246,23986021,23991736,24034977,24057008,23934970,23983296,24003105,23983814,24043960,23891344,24011363,24056265,24059522,24062574,24058128,24058293,24049577,24058780,24038425,24049569,23857950,23996830,23940237,23890959,24049820,24044124,23973490,23744176,24045469,23918597,24049567,23877023,24049573,23998056,24058812,23882685,24027649,23944779,24004644,23974595,24036948,24053866,24001373,24059897,24058380,24052245,24058861,24063702\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x860a6041dbed4108\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210526\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwjH_46W5sbxAhUVD-AKHU6hBdo\u003d\",\"frameworkUpdates\":{\"entityBatchUpdate\":{\"mutations\":[{\"entityKey\":\"EgcKBWFiY2RlIPYBKAE%3D\",\"type\":\"ENTITY_MUTATION_TYPE_REPLACE\",\"payload\":{\"offlineabilityEntity\":{\"key\":\"EgcKBWFiY2RlIPYBKAE%3D\",\"accessState\":\"OFFLINEABILITY_FEATURE_ACCESS_STATE_UNKNOWN\"}}}],\"timestamp\":{\"seconds\":\"1625311771\",\"nanos\":92746917}}}}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtEZDdTSEszdmlXSSiblIGHBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbXk3WU5od2t4OV9Gc0V6TXdFQ0FTQ1U2c2Z2Z3xBQ3Jtc0trRm9jNTQ3UXUydGFVdUJsYTVqQVptaUVsUHk5NmZTaTVRYjlQam5BVG1iaVVZLUs1aGlJQWpPdUtCSmdxVWg3VUVfQWdYR3Y0eTZ0SjFHUHF0YkI5OEhrbkRhU0tIWFUwUTh1Vm0ta09iVVVIeGVpbw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dabcde\\u0026list\\u003dRDabcde\",\"endpoint\": {\"clickTrackingParams\":\"IhMImKCOlubG8QIVjofeCh0Zow5zMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dabcde\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"abcde\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
+    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204\",\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23970530,24005646,23968386,24005602,23918597,24011119,23934970,23744176,24022914,23890959,23974595,23983296,23857950,24007246,9407156,24016478,1714250,24022308,23976696,24006670,24009750,24001373,24022875,24006795,24012117,23885487,23940238,23891346,24014268,24002697,23966208,23987676,23804281,23946420,23884386,23882502,24024495,23969934,24025870,23986021,23944779,24002011,23748146,24022616,23891344,24010576,23981192,24021968\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20210408.08.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xce87950a1c7b0640\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210408\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwijlIybx_bvAhWnF_EFHZCGD0U\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtjdWVaRlB4MmUxRSjauMyDBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbjU0XzV6c3N4VDlZNXpGU3RuVW5IZ18wSkdsd3xBQ3Jtc0tuMkpBMjhqbEY2WEc5VTd3Mk1XZ3pMWXdVMmhQZzlXY3pOY05mV0FQQURJLVh3U3NqVHFSLUpsdmxtZnJURkhrOXdhTFE5T2RCV3dXSHQ2UlZVRHJnRjRmYkpvN1BGUEI3TllwZktBbnZyQ29LRVZwZw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dabcde\\u0026list\\u003dRDabcde\",\"endpoint\": {\"clickTrackingParams\":\"IhMIs62Lm8f27wIVST_xBR3LIARZMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dabcde\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"abcde\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
     "latestUrl": "https://www.youtube.com/watch?v\u003dabcde\u0026list\u003dRDabcde\u0026pbj\u003d1"
   }
 }