From 8e27801183e9cbb713a27fe4b7305b349c9ee523 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Sep 2018 03:20:29 +0530 Subject: [PATCH] removed jackson and java 8 --- build.gradle | 4 +- extractor/build.gradle | 1 - .../extractors/YoutubeCommentsExtractor.java | 152 +++++------------- .../YoutubeCommentsInfoItemExtractor.java | 78 +++++++++ .../newpipe/extractor/utils/JsonUtils.java | 53 ++++++ .../java/org/schabi/newpipe/Downloader.java | 10 +- .../youtube/YoutubeCommentsExtractorTest.java | 9 +- .../extractor/utils/JsonUtilsTest.java | 47 ++++++ 8 files changed, 231 insertions(+), 123 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java diff --git a/build.gradle b/build.gradle index f8f6c8e90..9f25fc28e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ allprojects { apply plugin: 'java-library' - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 1.7 + targetCompatibility = 1.7 version 'v0.13.0' diff --git a/extractor/build.gradle b/extractor/build.gradle index 26430b9fa..1b7fbf001 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -6,7 +6,6 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.5' testImplementation 'junit:junit:4.12' } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index fb08e9b7a..259e52907 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -7,7 +7,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; @@ -21,9 +20,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; public class YoutubeCommentsExtractor extends CommentsExtractor { @@ -34,8 +36,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { private String title; private InfoItemsPage initPage; - private ObjectMapper mapper = new ObjectMapper(); - public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -57,16 +57,24 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { return initPage.getNextPageUrl(); } - private String getNextPageUrl(JsonNode ajaxJson) throws IOException, ExtractionException { - Optional element = Optional.ofNullable(ajaxJson.findValue("itemSectionContinuation")) - .map(e -> e.get("continuations")).map(e -> e.findValue("continuation")); - - if (element.isPresent()) { - return getNextPageUrl(element.get().asText()); - } else { - // no more comments + private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException { + + JsonArray arr; + try { + arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations"); + } catch (ParsingException e) { return ""; } + if(null == arr || arr.isEmpty()) { + return ""; + } + String continuation; + try { + continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); + } catch (ParsingException e) { + return ""; + } + return getNextPageUrl(continuation); } private String getNextPageUrl(String continuation) throws ParsingException { @@ -88,121 +96,35 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } String ajaxResponse = makeAjaxRequest(pageUrl); - JsonNode ajaxJson = mapper.readTree(ajaxResponse); + JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(ajaxResponse); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for comments", e); + } CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); collectCommentsFrom(collector, ajaxJson, pageUrl); return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson)); } - private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonNode ajaxJson, String pageUrl) { + private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException { - fetchTitle(ajaxJson); - List comments = ajaxJson.findValues("commentRenderer"); - comments.stream().forEach(c -> { - CommentsInfoItemExtractor extractor = new CommentsInfoItemExtractor() { - - @Override - public String getUrl() throws ParsingException { - return pageUrl; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return c.get("authorText").get("simpleText").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get author name", e); - } - } - - @Override - public String getPublishedTime() throws ParsingException { - try { - return c.get("publishedTimeText").get("runs").get(0).get("text").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get publishedTimeText", e); - } - } - - @Override - public Integer getLikeCount() throws ParsingException { - try { - return c.get("likeCount").intValue(); - } catch (Exception e) { - throw new ParsingException("Could not get like count", e); - } - } - - @Override - public String getCommentText() throws ParsingException { - try { - if (null != c.get("contentText").get("simpleText")) { - return c.get("contentText").get("simpleText").asText(); - } else { - return c.get("contentText").get("runs").get(0).get("text").asText(); - } - } catch (Exception e) { - throw new ParsingException("Could not get comment text", e); - } - } - - @Override - public String getCommentId() throws ParsingException { - try { - return c.get("commentId").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get comment id", e); - } - } - - @Override - public String getAuthorThumbnail() throws ParsingException { - try { - return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get author thumbnail", e); - } - } - - @Override - public String getAuthorName() throws ParsingException { - try { - return c.get("authorText").get("simpleText").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get author name", e); - } - } - - @Override - public String getAuthorEndpoint() throws ParsingException { - try { - return "https://youtube.com" - + c.get("authorEndpoint").get("browseEndpoint").get("canonicalBaseUrl").asText(); - } catch (Exception e) { - throw new ParsingException("Could not get author endpoint", e); - } - } - }; - + JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents"); + fetchTitle(contents); + List comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); + + for(JsonObject c: comments) { + CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl); collector.commit(extractor); - }); + } } - private void fetchTitle(JsonNode ajaxJson) { + private void fetchTitle(JsonArray contents) { if(null == title) { try { - title = ajaxJson.findValue("commentTargetTitle").get("simpleText").asText(); + title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText"); } catch (Exception e) { title = "Youtube Comments"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java new file mode 100644 index 000000000..f6507f153 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -0,0 +1,78 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{ + + private final JsonObject json; + private final String url; + + public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) { + this.json = json; + this.url = url; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); + return JsonUtils.getValue(arr.getObject(2), "url"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getValue(json, "authorText.simpleText"); + } + + @Override + public String getPublishedTime() throws ParsingException { + JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs"); + return JsonUtils.getValue(arr.getObject(0), "text"); + } + + @Override + public Integer getLikeCount() throws ParsingException { + return JsonUtils.getValue(json, "likeCount"); + } + + @Override + public String getCommentText() throws ParsingException { + try { + return JsonUtils.getValue(json, "contentText.simpleText"); + } catch (Exception e) { + JsonArray arr = JsonUtils.getValue(json, "contentText.runs"); + return JsonUtils.getValue(arr.getObject(0), "text"); + } + } + + @Override + public String getCommentId() throws ParsingException { + return JsonUtils.getValue(json, "commentId"); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); + return JsonUtils.getValue(arr.getObject(2), "url"); + } + + @Override + public String getAuthorName() throws ParsingException { + return JsonUtils.getValue(json, "authorText.simpleText"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl"); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java new file mode 100644 index 000000000..8aa5cea4b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -0,0 +1,53 @@ +package org.schabi.newpipe.extractor.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +public class JsonUtils { + + private JsonUtils() { + } + + @Nonnull + public static T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + + List keys = Arrays.asList(path.split("\\.")); + object = getObject(object, keys.subList(0, keys.size() - 1)); + if (null == object) throw new ParsingException("Unable to get " + path); + T result = (T) object.get(keys.get(keys.size() - 1)); + if(null == result) throw new ParsingException("Unable to get " + path); + return result; + } + + + @Nonnull + public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { + + List result = new ArrayList<>(); + for (int i = 0; i < array.size(); i++) { + JsonObject obj = array.getObject(i); + result.add((T)getValue(obj, path)); + } + return result; + } + + @Nullable + private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List keys) { + JsonObject result = object; + for (String key : keys) { + result = result.getObject(key); + if (null == result) break; + } + return result; + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/Downloader.java b/extractor/src/test/java/org/schabi/newpipe/Downloader.java index 385d896b1..4745a5d1a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/Downloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/Downloader.java @@ -177,7 +177,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); for (Map.Entry> pair : requestHeaders.entrySet()) { - pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value)); + for(String value: pair.getValue()) { + con.addRequestProperty(pair.getKey(), value); + } } String responseBody = dl(con); return new DownloadResponse(responseBody, con.getHeaderFields()); @@ -185,7 +187,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { @Override public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException { - return get(siteUrl, Collections.emptyMap()); + return get(siteUrl, Collections.EMPTY_MAP); } @Override @@ -195,7 +197,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); con.setRequestMethod("POST"); for (Map.Entry> pair : requestHeaders.entrySet()) { - pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value)); + for(String value: pair.getValue()) { + con.addRequestProperty(pair.getKey(), value); + } } // set fields to default if not set already setDefaults(con); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index f811e1124..afef25145 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -41,7 +41,7 @@ public class YoutubeCommentsExtractorTest { assertTrue(result); } - + @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; @@ -64,6 +64,11 @@ public class YoutubeCommentsExtractorTest { } private boolean findInComments(List comments, String comment) { - return comments.stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent(); + for(CommentsInfoItem c: comments) { + if(c.getCommentText().contains(comment)) { + return true; + } + } + return false; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java new file mode 100644 index 000000000..b44d3ee9c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.extractor.utils; + + +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + + +public class JsonUtilsTest { + + @Test + public void testGetValueFlat() throws JsonParserException, ParsingException { + JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); + assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); + } + + @Test + public void testGetValueNested() throws JsonParserException, ParsingException { + JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); + assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2"))); + } + + @Test + public void testGetArray() throws JsonParserException, ParsingException { + JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); + JsonArray arr = JsonUtils.getValue(obj, "batters.batter"); + assertTrue(!arr.isEmpty()); + } + + @Test + public void testGetValues() throws JsonParserException, ParsingException { + JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); + JsonArray arr = JsonUtils.getValue(obj, "topping"); + List types = JsonUtils.getValues(arr, "type"); + assertTrue(types.contains("Chocolate with Sprinkles")); + + } + +}