Improve getClientVersion()

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
-import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -68,10 +66,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");
@@ -222,10 +218,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -43,10 +43,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");
@@ -202,10 +200,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
-import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
@@ -61,10 +59,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");
@@ -130,10 +126,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        // Use the hardcoded client version first to get JSON with a structure we know
-        // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
         headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
         final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
         if (response.length() < 50) { // ensure to have a valid response
             throw new ParsingException("Could not parse json data for next streams");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -5,6 +5,7 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -16,7 +17,13 @@ import java.net.URL;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -43,7 +50,9 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
-    public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
+    private static String clientVersion;
+
 
     private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@@ -162,55 +171,83 @@ public class YoutubeParsingHelper {
 
     /**
      * Get the client version from a page
-     * @param initialData
-     * @param html The page HTML
      * @return
      * @throws ParsingException
      */
-    public static String getClientVersion(JsonObject initialData, String html) throws ParsingException {
+    public static String getClientVersion() throws ParsingException {
-        if (initialData == null) initialData = getInitialData(html);
+        if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
-        JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
-        String shortClientVersion = null;
 
-        // try to get version from initial data first
+        // Test if hard-coded client version is valid
-        for (Object service : serviceTrackingParams) {
+        try {
-            JsonObject s = (JsonObject) service;
+            final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
-            if (s.getString("service").equals("CSI")) {
+
-                JsonArray params = s.getArray("params");
+            Map<String, List<String>> headers = new HashMap<>();
-                for (Object param: params) {
+            headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-                    JsonObject p = (JsonObject) param;
+            headers.put("X-YouTube-Client-Version",
-                    String key = p.getString("key");
+                    Collections.singletonList(HARDCODED_CLIENT_VERSION));
-                    if (key != null && key.equals("cver")) {
+            final String response = getDownloader().get(url, headers).responseBody();
-                        return p.getString("value");
+            if (response.length() > 50) { // ensure to have a valid response
+                clientVersion = HARDCODED_CLIENT_VERSION;
+                return clientVersion;
+            }
+        } catch (Exception ignored) {}
+
+        // Try extracting it from YouTube's website otherwise
+        try {
+            final String url = "https://www.youtube.com/results?search_query=test";
+            final String html = getDownloader().get(url).responseBody();
+            JsonObject initialData = getInitialData(html);
+            JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
+            String shortClientVersion = null;
+
+            // try to get version from initial data first
+            for (Object service : serviceTrackingParams) {
+                JsonObject s = (JsonObject) service;
+                if (s.getString("service").equals("CSI")) {
+                    JsonArray params = s.getArray("params");
+                    for (Object param : params) {
+                        JsonObject p = (JsonObject) param;
+                        String key = p.getString("key");
+                        if (key != null && key.equals("cver")) {
+                            clientVersion = p.getString("value");
+                            return clientVersion;
+                        }
                     }
-                }
+                } else if (s.getString("service").equals("ECATCHER")) {
-            } else if (s.getString("service").equals("ECATCHER")) {
+                    // fallback to get a shortened client version which does not contain the last do digits
-                // fallback to get a shortened client version which does not contain the last do digits
+                    JsonArray params = s.getArray("params");
-                JsonArray params = s.getArray("params");
+                    for (Object param : params) {
-                for (Object param: params) {
+                        JsonObject p = (JsonObject) param;
-                    JsonObject p = (JsonObject) param;
+                        String key = p.getString("key");
-                    String key = p.getString("key");
+                        if (key != null && key.equals("client.version")) {
-                    if (key != null && key.equals("client.version")) {
+                            shortClientVersion = p.getString("value");
-                        shortClientVersion = p.getString("value");
+                        }
                     }
                 }
             }
-        }
 
-        String clientVersion;
+            String contextClientVersion;
-        String[] patterns = {
+            String[] patterns = {
-                "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
+                    "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
-                "innertube_context_client_version\":\"([0-9\\.]+?)\"",
+                    "innertube_context_client_version\":\"([0-9\\.]+?)\"",
-                "client.version=([0-9\\.]+)"
+                    "client.version=([0-9\\.]+)"
-        };
+            };
-        for (String pattern: patterns) {
+            for (String pattern : patterns) {
-            try {
+                try {
-                clientVersion = Parser.matchGroup1(pattern, html);
+                    contextClientVersion = Parser.matchGroup1(pattern, html);
-                if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
+                    if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
-            } catch (Exception ignored) {}
+                        clientVersion = contextClientVersion;
-        }
+                        return clientVersion;
+                    }
+                } catch (Exception ignored) {
+                }
+            }
 
-        if (shortClientVersion != null) return shortClientVersion;
+            if (shortClientVersion != null) {
+                clientVersion = shortClientVersion;
+                return clientVersion;
+            }
+        } catch (Exception ignored) {}
 
         throw new ParsingException("Could not get client version");
     }