From d8280ce0da0651165b54ed532517e14749ea037e Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sun, 28 Apr 2019 17:03:17 -0300 Subject: [PATCH] [YouTube] Parse watching count in live streams items --- .../extractors/YoutubeStreamExtractor.java | 53 +++++++++++++++++++ .../YoutubeStreamInfoItemExtractor.java | 46 +++++++++++----- .../YoutubeStreamExtractorLivestreamTest.java | 2 +- 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 242395371..092fd41ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -304,12 +304,65 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getViewCount() throws ParsingException { assertPageFetched(); try { + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + return getLiveStreamWatchingCount(); + } + return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); } catch (Exception e) {//todo: find fallback method throw new ParsingException("Could not get number of views", e); } } + private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException { + // https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key= + String innerTubeKey = null, clientVersion = null; + if (playerArgs != null && !playerArgs.isEmpty()) { + innerTubeKey = playerArgs.getString("innertube_api_key"); + clientVersion = playerArgs.getString("innertube_context_client_version"); + } else if (!videoInfoPage.isEmpty()) { + innerTubeKey = videoInfoPage.get("innertube_api_key"); + clientVersion = videoInfoPage.get("innertube_context_client_version"); + } + + if (innerTubeKey == null || innerTubeKey.isEmpty()) { + throw new ExtractionException("Couldn't get innerTube key"); + } + + if (clientVersion == null || clientVersion.isEmpty()) { + throw new ExtractionException("Couldn't get innerTube client version"); + } + + final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey; + final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" + + ",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8"); + final Response response = getDownloader().execute(Request.newBuilder() + .post(metadataUrl, dataBody) + .addHeader("Content-Type", "application/json") + .build()); + final JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + + for (Object actionEntry : jsonObject.getArray("actions")) { + if (!(actionEntry instanceof JsonObject)) continue; + final JsonObject entry = (JsonObject) actionEntry; + + final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null); + if (updateViewershipAction == null) continue; + + final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs"); + if (viewCountRuns.isEmpty()) continue; + + final JsonObject textObject = viewCountRuns.getObject(0); + if (!textObject.has("text")) { + throw new ExtractionException("Response don't have \"text\" element"); + } + + return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text"))); + } + + throw new ExtractionException("Could not find correct results in response"); + } + @Override public long getLikeCount() throws ParsingException { assertPageFetched(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 162e2e49f..46afe7ab0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -3,10 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; @@ -141,6 +141,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getTextualUploadDate() throws ParsingException { + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + return null; + } + if (cachedUploadDate != null) { return cachedUploadDate; } @@ -160,9 +164,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public Calendar getUploadDate() throws ParsingException { + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + return null; + } + String textualUploadDate = getTextualUploadDate(); - if (timeAgoParser != null - && textualUploadDate != null && !"".equals(textualUploadDate)) { + if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { return timeAgoParser.parse(textualUploadDate); } else { return null; @@ -172,24 +179,35 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getViewCount() throws ParsingException { String input; - try { - // TODO: Return the actual live stream's watcher count - // -1 for no view count - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); - if (meta == null) return -1; + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + Element meta = item.select("ul[class=\"yt-lockup-meta-info\"]").first(); + if (meta == null) return 0; - // This case can happen if google releases a special video - if(meta.select("li").size() < 2) return -1; + final Elements li = meta.select("li"); + if (li.isEmpty()) return 0; - input = meta.select("li").get(1).text(); + input = li.first().text(); + } else { + try { + Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); + if (meta == null) return -1; - } catch (IndexOutOfBoundsException e) { - throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); + // This case can happen if google releases a special video + if (meta.select("li").size() < 2) return -1; + + input = meta.select("li").get(1).text(); + } catch (IndexOutOfBoundsException e) { + throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); + } + } + + if (input == null) { + throw new ParsingException("Input is null"); } try { + return Long.parseLong(Utils.removeNonDigitCharacters(input)); } catch (NumberFormatException e) { // if this happens the video probably has no views diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 408c29a3f..eb2541edf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -68,7 +68,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetViewCount() throws ParsingException { long count = extractor.getViewCount(); - assertTrue(Long.toString(count), count >= 7148995); + assertTrue(Long.toString(count), count > -1); } @Test