From 985c3ec877fa8e62073bb1fc81135e08ac33fda2 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 15:22:59 +0100 Subject: [PATCH] Improve getClientVersion() --- .../extractors/YoutubeChannelExtractor.java | 10 +- .../extractors/YoutubePlaylistExtractor.java | 8 +- .../extractors/YoutubeSearchExtractor.java | 10 +- .../linkHandler/YoutubeParsingHelper.java | 117 ++++++++++++------ 4 files changed, 83 insertions(+), 62 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 3b1a9e66f..41b0de1da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -68,10 +66,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -222,10 +218,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 476200300..84af2146d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -43,10 +43,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -202,10 +200,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 2396498fe..7ec0e5cc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; @@ -61,10 +59,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -130,10 +126,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 51347d423..10caa5091 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -16,7 +17,13 @@ import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; /* * Created by Christian Schabesberger on 02.03.16. @@ -43,7 +50,9 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static String clientVersion; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -162,55 +171,83 @@ public class YoutubeParsingHelper { /** * Get the client version from a page - * @param initialData - * @param html The page HTML * @return * @throws ParsingException */ - public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { - if (initialData == null) initialData = getInitialData(html); - JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); - String shortClientVersion = null; + public static String getClientVersion() throws ParsingException { + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - // try to get version from initial data first - for (Object service : serviceTrackingParams) { - JsonObject s = (JsonObject) service; - if (s.getString("service").equals("CSI")) { - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("cver")) { - return p.getString("value"); + // Test if hard-coded client version is valid + try { + final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + clientVersion = HARDCODED_CLIENT_VERSION; + return clientVersion; + } + } catch (Exception ignored) {} + + // Try extracting it from YouTube's website otherwise + try { + final String url = "https://www.youtube.com/results?search_query=test"; + final String html = getDownloader().get(url).responseBody(); + JsonObject initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + clientVersion = p.getString("value"); + return clientVersion; + } } - } - } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last do digits - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("client.version")) { - shortClientVersion = p.getString("value"); + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last do digits + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } } } } - } - String clientVersion; - String[] patterns = { - "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", - "innertube_context_client_version\":\"([0-9\\.]+?)\"", - "client.version=([0-9\\.]+)" - }; - for (String pattern: patterns) { - try { - clientVersion = Parser.matchGroup1(pattern, html); - if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - } catch (Exception ignored) {} - } + String contextClientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern : patterns) { + try { + contextClientVersion = Parser.matchGroup1(pattern, html); + if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + clientVersion = contextClientVersion; + return clientVersion; + } + } catch (Exception ignored) { + } + } - if (shortClientVersion != null) return shortClientVersion; + if (shortClientVersion != null) { + clientVersion = shortClientVersion; + return clientVersion; + } + } catch (Exception ignored) {} throw new ParsingException("Could not get client version"); }