Add getClientVersion() and HARDCODED_CLIENT_VERSION to YouTubeParsingHelper

Prefer hardcoded client version above the current one when making requests to retrieve the same JSON structure for each request.

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -211,14 +211,30 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         JsonArray ajaxJson;
-        try {
+
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-            headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow
+        try {
+            // Use the hardcoded client version first to get JSON with a structure we know
+            headers.put("X-YouTube-Client-Version",
+                    Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
             final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
+            if (response.length() > 50) { // ensure to have a valid response
+                throw new ParsingException("Could not parse json data for next streams");
+            }
             ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException pe) {
+        } catch (Exception e) {
-            throw new ParsingException("Could not parse json data for next streams", pe);
+            try {
+                headers.put("X-YouTube-Client-Version",
+                        Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
+                final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
+                if (response.length() > 50) { // ensure to have a valid response
+                    throw new ParsingException("Could not parse json data for next streams");
+                }
+                ajaxJson = JsonParser.array().from(response);
+            } catch (JsonParserException ignored) {
+                throw new ParsingException("Could not parse json data for next streams", e);
+            }
         }
 
         JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -198,14 +198,30 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         JsonArray ajaxJson;
-        try {
+
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-            headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow
+        try {
+            // Use the hardcoded client version first to get JSON with a structure we know
+            headers.put("X-YouTube-Client-Version",
+                    Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
             final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
+            if (response.length() > 50) { // ensure to have a valid response
+                throw new ParsingException("Could not parse json data for next streams");
+            }
             ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException pe) {
+        } catch (Exception e) {
-            throw new ParsingException("Could not parse json data for next streams", pe);
+            try {
+                headers.put("X-YouTube-Client-Version",
+                        Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
+                final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
+                if (response.length() > 50) { // ensure to have a valid response
+                    throw new ParsingException("Could not parse json data for next streams");
+                }
+                ajaxJson = JsonParser.array().from(response);
+            } catch (JsonParserException ignored) {
+                throw new ParsingException("Could not parse json data for next streams", e);
+            }
         }
 
         JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
@@ -42,6 +43,8 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
+    public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
+
     private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
 
@@ -157,4 +160,59 @@ public class YoutubeParsingHelper {
         }
     }
 
+    /**
+     * Get the client version from a page
+     * @param initialData
+     * @param html The page HTML
+     * @return
+     * @throws ParsingException
+     */
+    public static String getClientVersion(JsonObject initialData, String html) throws ParsingException {
+        if (initialData == null) initialData = getInitialData(html);
+        JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
+        String shortClientVersion = null;
+
+        // try to get version from initial data first
+        for (Object service : serviceTrackingParams) {
+            JsonObject s = (JsonObject) service;
+            if (s.getString("service").equals("CSI")) {
+                JsonArray params = s.getArray("params");
+                for (Object param: params) {
+                    JsonObject p = (JsonObject) param;
+                    String key = p.getString("key");
+                    if (key != null && key.equals("cver")) {
+                        return p.getString("value");
+                    }
+                }
+            } else if (s.getString("service").equals("ECATCHER")) {
+                // fallback to get a shortened client version which does not contain the last do digits
+                JsonArray params = s.getArray("params");
+                for (Object param: params) {
+                    JsonObject p = (JsonObject) param;
+                    String key = p.getString("key");
+                    if (key != null && key.equals("client.version")) {
+                        shortClientVersion = p.getString("value");
+                    }
+                }
+            }
+        }
+
+        String clientVersion;
+        String[] patterns = {
+                "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
+                "innertube_context_client_version\":\"([0-9\\.]+?)\"",
+                "client.version=([0-9\\.]+)"
+        };
+        for (String pattern: patterns) {
+            try {
+                clientVersion = Parser.matchGroup1(pattern, html);
+                if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
+            } catch (Exception ignored) {}
+        }
+
+        if (shortClientVersion != null) return shortClientVersion;
+
+        throw new ParsingException("Could not get client version");
+    }
+
 }