removed jackson and java 8

build.gradle

@@ -1,7 +1,7 @@
 allprojects {
     apply plugin: 'java-library'
-    sourceCompatibility = 1.8
+    sourceCompatibility = 1.7
-    targetCompatibility = 1.8
+    targetCompatibility = 1.7
 
     version 'v0.13.0'
 

extractor/build.gradle

@@ -6,7 +6,6 @@ dependencies {
     implementation 'org.mozilla:rhino:1.7.7.1'
     implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
     implementation 'org.nibor.autolink:autolink:0.8.0'
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.5'
 
     testImplementation 'junit:junit:4.12'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -7,7 +7,6 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 
 import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
@@ -21,9 +20,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import com.fasterxml.jackson.databind.JsonNode;
+import com.grack.nanojson.JsonArray;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
 
 public class YoutubeCommentsExtractor extends CommentsExtractor {
 
@@ -34,8 +36,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
     private String title;
     private InfoItemsPage<CommentsInfoItem> initPage;
 
-    private ObjectMapper mapper = new ObjectMapper();
-
     public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
         super(service, uiHandler);
     }
@@ -57,16 +57,24 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
         return initPage.getNextPageUrl();
     }
 
-    private String getNextPageUrl(JsonNode ajaxJson) throws IOException, ExtractionException {
+    private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException {
-        Optional<JsonNode> element = Optional.ofNullable(ajaxJson.findValue("itemSectionContinuation"))
+        
-                .map(e -> e.get("continuations")).map(e -> e.findValue("continuation"));
+        JsonArray arr;
-
+        try {
-        if (element.isPresent()) {
+            arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
-            return getNextPageUrl(element.get().asText());
+        } catch (ParsingException e) {
-        } else {
-            // no more comments
             return "";
         }
+        if(null == arr || arr.isEmpty()) {
+            return "";
+        }
+        String continuation;
+        try {
+            continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
+        } catch (ParsingException e) {
+            return "";
+        }
+        return getNextPageUrl(continuation);
     }
 
     private String getNextPageUrl(String continuation) throws ParsingException {
@@ -88,121 +96,35 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
             throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
         }
         String ajaxResponse = makeAjaxRequest(pageUrl);
-        JsonNode ajaxJson = mapper.readTree(ajaxResponse);
+        JsonObject ajaxJson;
+        try {
+            ajaxJson = JsonParser.object().from(ajaxResponse);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json data for comments", e);
+        }
         CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
         collectCommentsFrom(collector, ajaxJson, pageUrl);
         return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson));
     }
 
-    private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonNode ajaxJson, String pageUrl) {
+    private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
         
-        fetchTitle(ajaxJson);
         
-        List<JsonNode> comments = ajaxJson.findValues("commentRenderer");
+        JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
-        comments.stream().forEach(c -> {
+        fetchTitle(contents);
-            CommentsInfoItemExtractor extractor = new CommentsInfoItemExtractor() {
+        List<JsonObject> comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
-
+        
-                @Override
+        for(JsonObject c: comments) {
-                public String getUrl() throws ParsingException {
+            CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl);
-                    return pageUrl;
-                }
-
-                @Override
-                public String getThumbnailUrl() throws ParsingException {
-                    try {
-                        return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get thumbnail url", e);
-                    }
-                }
-
-                @Override
-                public String getName() throws ParsingException {
-                    try {
-                        return c.get("authorText").get("simpleText").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get author name", e);
-                    }
-                }
-
-                @Override
-                public String getPublishedTime() throws ParsingException {
-                    try {
-                        return c.get("publishedTimeText").get("runs").get(0).get("text").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get publishedTimeText", e);
-                    }
-                }
-
-                @Override
-                public Integer getLikeCount() throws ParsingException {
-                    try {
-                        return c.get("likeCount").intValue();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get like count", e);
-                    }
-                }
-
-                @Override
-                public String getCommentText() throws ParsingException {
-                    try {
-                        if (null != c.get("contentText").get("simpleText")) {
-                            return c.get("contentText").get("simpleText").asText();
-                        } else {
-                            return c.get("contentText").get("runs").get(0).get("text").asText();
-                        }
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get comment text", e);
-                    }
-                }
-
-                @Override
-                public String getCommentId() throws ParsingException {
-                    try {
-                        return c.get("commentId").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get comment id", e);
-                    }
-                }
-
-                @Override
-                public String getAuthorThumbnail() throws ParsingException {
-                    try {
-                        return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get author thumbnail", e);
-                    }
-                }
-
-                @Override
-                public String getAuthorName() throws ParsingException {
-                    try {
-                        return c.get("authorText").get("simpleText").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get author name", e);
-                    }
-                }
-
-                @Override
-                public String getAuthorEndpoint() throws ParsingException {
-                    try {
-                        return "https://youtube.com"
-                                + c.get("authorEndpoint").get("browseEndpoint").get("canonicalBaseUrl").asText();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get author endpoint", e);
-                    }
-                }
-            };
-
             collector.commit(extractor);
-        });
+        }
 
     }
 
-    private void fetchTitle(JsonNode ajaxJson) {
+    private void fetchTitle(JsonArray contents) {
         if(null == title) {
             try {
-                title = ajaxJson.findValue("commentTargetTitle").get("simpleText").asText();
+                title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
             } catch (Exception e) {
                 title = "Youtube Comments";
             }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -0,0 +1,78 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+
+public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{
+    
+    private final JsonObject json;
+    private final String url;
+    
+    public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
+        this.json = json;
+        this.url = url;
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        return url;
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
+        return JsonUtils.getValue(arr.getObject(2), "url");
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return JsonUtils.getValue(json, "authorText.simpleText");
+    }
+
+    @Override
+    public String getPublishedTime() throws ParsingException {
+        JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs");
+        return JsonUtils.getValue(arr.getObject(0), "text");
+    }
+
+    @Override
+    public Integer getLikeCount() throws ParsingException {
+        return JsonUtils.getValue(json, "likeCount");
+    }
+
+    @Override
+    public String getCommentText() throws ParsingException {
+        try {
+            return JsonUtils.getValue(json, "contentText.simpleText");
+        } catch (Exception e) {
+            JsonArray arr = JsonUtils.getValue(json, "contentText.runs");
+            return JsonUtils.getValue(arr.getObject(0), "text");
+        }
+    }
+
+    @Override
+    public String getCommentId() throws ParsingException {
+        return JsonUtils.getValue(json, "commentId");
+    }
+
+    @Override
+    public String getAuthorThumbnail() throws ParsingException {
+        JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
+        return JsonUtils.getValue(arr.getObject(2), "url");
+    }
+
+    @Override
+    public String getAuthorName() throws ParsingException {
+        return JsonUtils.getValue(json, "authorText.simpleText");
+    }
+
+    @Override
+    public String getAuthorEndpoint() throws ParsingException {
+            return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
+    }
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -0,0 +1,53 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+
+public class JsonUtils {
+
+    private JsonUtils() {
+    }
+    
+    @Nonnull
+    public static <T> T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+
+        List<String> keys = Arrays.asList(path.split("\\."));
+        object = getObject(object, keys.subList(0, keys.size() - 1));
+        if (null == object) throw new ParsingException("Unable to get " + path);
+        T result = (T) object.get(keys.get(keys.size() - 1));
+        if(null == result) throw new ParsingException("Unable to get " + path);
+        return result;
+    }
+    
+
+    @Nonnull
+    public static <T> List<T> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
+        
+        List<T> result = new ArrayList<>();
+        for (int i = 0; i < array.size(); i++) {
+            JsonObject obj = array.getObject(i);
+            result.add((T)getValue(obj, path));
+        }
+        return result;
+    }
+
+    @Nullable
+    private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List<String> keys) {
+        JsonObject result = object;
+        for (String key : keys) {
+            result = result.getObject(key);
+            if (null == result) break;
+        }
+        return result;
+    }
+
+}

extractor/src/test/java/org/schabi/newpipe/Downloader.java

@@ -177,7 +177,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
-            pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
+            for(String value: pair.getValue()) {
+                con.addRequestProperty(pair.getKey(), value);
+            }
         }
         String responseBody = dl(con);
         return new DownloadResponse(responseBody, con.getHeaderFields());
@@ -185,7 +187,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
 
     @Override
     public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
-        return get(siteUrl, Collections.emptyMap());
+        return get(siteUrl, Collections.EMPTY_MAP);
     }
 
     @Override
@@ -195,7 +197,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         con.setRequestMethod("POST");
         for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
-            pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
+            for(String value: pair.getValue()) {
+                con.addRequestProperty(pair.getKey(), value);
+            }
         }
         // set fields to default if not set already
         setDefaults(con);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -41,7 +41,7 @@ public class YoutubeCommentsExtractorTest {
 
         assertTrue(result);
     }
-    
+
     @Test
     public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
         boolean result = false;
@@ -64,6 +64,11 @@ public class YoutubeCommentsExtractorTest {
     }
 
     private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
-        return comments.stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
+        for(CommentsInfoItem c: comments) {
+            if(c.getCommentText().contains(comment)) {
+                return true;
+            }
+        }
+        return false;
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java

@@ -0,0 +1,47 @@
+package org.schabi.newpipe.extractor.utils;
+
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.junit.Test;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+
+
+public class JsonUtilsTest {
+    
+    @Test
+    public void testGetValueFlat() throws JsonParserException, ParsingException {
+        JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
+        assertTrue("John".equals(JsonUtils.getValue(obj, "name")));        
+    }
+    
+    @Test
+    public void testGetValueNested() throws JsonParserException, ParsingException {
+        JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
+        assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2")));
+    }
+    
+    @Test
+    public void testGetArray() throws JsonParserException, ParsingException {
+        JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
+        JsonArray arr = JsonUtils.getValue(obj, "batters.batter");
+        assertTrue(!arr.isEmpty());
+    }
+    
+    @Test
+    public void testGetValues() throws JsonParserException, ParsingException {
+        JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
+        JsonArray arr = JsonUtils.getValue(obj, "topping");
+        List<String> types = JsonUtils.getValues(arr, "type");
+        assertTrue(types.contains("Chocolate with Sprinkles"));
+        
+    }
+
+}