be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

removed jackson and java 8

This commit is contained in:
Ritvik Saraf 2018-09-26 03:20:29 +05:30
parent c1199c8fcf
commit 8e27801183
8 changed files with 231 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle
View File

@ -1,7 +1,7 @@
allprojects {
apply plugin: 'java-library'
sourceCompatibility = 1.8
targetCompatibility = 1.8
sourceCompatibility = 1.7
targetCompatibility = 1.7
version 'v0.13.0'

1
extractor/build.gradle
View File

@ -6,7 +6,6 @@ dependencies {
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.5'
testImplementation 'junit:junit:4.12'
}

152
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -7,7 +7,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
@ -21,9 +20,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -34,8 +36,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String title;
private InfoItemsPage<CommentsInfoItem> initPage;
private ObjectMapper mapper = new ObjectMapper();
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@ -57,16 +57,24 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return initPage.getNextPageUrl();
}
private String getNextPageUrl(JsonNode ajaxJson) throws IOException, ExtractionException {
Optional<JsonNode> element = Optional.ofNullable(ajaxJson.findValue("itemSectionContinuation"))
.map(e -> e.get("continuations")).map(e -> e.findValue("continuation"));
if (element.isPresent()) {
return getNextPageUrl(element.get().asText());
} else {
// no more comments
private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException {
JsonArray arr;
try {
arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
} catch (ParsingException e) {
return "";
}
if(null == arr || arr.isEmpty()) {
return "";
}
String continuation;
try {
continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
} catch (ParsingException e) {
return "";
}
return getNextPageUrl(continuation);
}
private String getNextPageUrl(String continuation) throws ParsingException {
@ -88,121 +96,35 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
String ajaxResponse = makeAjaxRequest(pageUrl);
JsonNode ajaxJson = mapper.readTree(ajaxResponse);
JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(ajaxResponse);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for comments", e);
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectCommentsFrom(collector, ajaxJson, pageUrl);
return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson));
}
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonNode ajaxJson, String pageUrl) {
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
fetchTitle(ajaxJson);
List<JsonNode> comments = ajaxJson.findValues("commentRenderer");
comments.stream().forEach(c -> {
CommentsInfoItemExtractor extractor = new CommentsInfoItemExtractor() {
@Override
public String getUrl() throws ParsingException {
return pageUrl;
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
try {
return c.get("authorText").get("simpleText").asText();
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
}
@Override
public String getPublishedTime() throws ParsingException {
try {
return c.get("publishedTimeText").get("runs").get(0).get("text").asText();
} catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
}
@Override
public Integer getLikeCount() throws ParsingException {
try {
return c.get("likeCount").intValue();
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
}
@Override
public String getCommentText() throws ParsingException {
try {
if (null != c.get("contentText").get("simpleText")) {
return c.get("contentText").get("simpleText").asText();
} else {
return c.get("contentText").get("runs").get(0).get("text").asText();
}
} catch (Exception e) {
throw new ParsingException("Could not get comment text", e);
}
}
@Override
public String getCommentId() throws ParsingException {
try {
return c.get("commentId").asText();
} catch (Exception e) {
throw new ParsingException("Could not get comment id", e);
}
}
@Override
public String getAuthorThumbnail() throws ParsingException {
try {
return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
} catch (Exception e) {
throw new ParsingException("Could not get author thumbnail", e);
}
}
@Override
public String getAuthorName() throws ParsingException {
try {
return c.get("authorText").get("simpleText").asText();
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
}
@Override
public String getAuthorEndpoint() throws ParsingException {
try {
return "https://youtube.com"
+ c.get("authorEndpoint").get("browseEndpoint").get("canonicalBaseUrl").asText();
} catch (Exception e) {
throw new ParsingException("Could not get author endpoint", e);
}
}
};
JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
fetchTitle(contents);
List<JsonObject> comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
for(JsonObject c: comments) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl);
collector.commit(extractor);
});
}
}
private void fetchTitle(JsonNode ajaxJson) {
private void fetchTitle(JsonArray contents) {
if(null == title) {
try {
title = ajaxJson.findValue("commentTargetTitle").get("simpleText").asText();
title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
} catch (Exception e) {
title = "Youtube Comments";
}

78
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java Normal file
View File

@ -0,0 +1,78 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{
private final JsonObject json;
private final String url;
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
this.json = json;
this.url = url;
}
@Override
public String getUrl() throws ParsingException {
return url;
}
@Override
public String getThumbnailUrl() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return JsonUtils.getValue(arr.getObject(2), "url");
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText");
}
@Override
public String getPublishedTime() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs");
return JsonUtils.getValue(arr.getObject(0), "text");
}
@Override
public Integer getLikeCount() throws ParsingException {
return JsonUtils.getValue(json, "likeCount");
}
@Override
public String getCommentText() throws ParsingException {
try {
return JsonUtils.getValue(json, "contentText.simpleText");
} catch (Exception e) {
JsonArray arr = JsonUtils.getValue(json, "contentText.runs");
return JsonUtils.getValue(arr.getObject(0), "text");
}
}
@Override
public String getCommentId() throws ParsingException {
return JsonUtils.getValue(json, "commentId");
}
@Override
public String getAuthorThumbnail() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return JsonUtils.getValue(arr.getObject(2), "url");
}
@Override
public String getAuthorName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText");
}
@Override
public String getAuthorEndpoint() throws ParsingException {
return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
}
}

53
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java Normal file
View File

@ -0,0 +1,53 @@
package org.schabi.newpipe.extractor.utils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
public class JsonUtils {
private JsonUtils() {
}
@Nonnull
public static <T> T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path);
T result = (T) object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path);
return result;
}
@Nonnull
public static <T> List<T> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<T> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i);
result.add((T)getValue(obj, path));
}
return result;
}
@Nullable
private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List<String> keys) {
JsonObject result = object;
for (String key : keys) {
result = result.getObject(key);
if (null == result) break;
}
return result;
}
}

10
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -177,7 +177,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
String responseBody = dl(con);
return new DownloadResponse(responseBody, con.getHeaderFields());
@ -185,7 +187,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
@Override
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
return get(siteUrl, Collections.emptyMap());
return get(siteUrl, Collections.EMPTY_MAP);
}
@Override
@ -195,7 +197,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
// set fields to default if not set already
setDefaults(con);

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -41,7 +41,7 @@ public class YoutubeCommentsExtractorTest {
assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
@ -64,6 +64,11 @@ public class YoutubeCommentsExtractorTest {
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
return comments.stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
for(CommentsInfoItem c: comments) {
if(c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}

47
extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java Normal file
View File

@ -0,0 +1,47 @@
package org.schabi.newpipe.extractor.utils;
import static org.junit.Assert.assertTrue;
import java.util.List;
import org.junit.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class JsonUtilsTest {
@Test
public void testGetValueFlat() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
assertTrue("John".equals(JsonUtils.getValue(obj, "name")));
}
@Test
public void testGetValueNested() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2")));
}
@Test
public void testGetArray() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "batters.batter");
assertTrue(!arr.isEmpty());
}
@Test
public void testGetValues() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "topping");
List<String> types = JsonUtils.getValues(arr, "type");
assertTrue(types.contains("Chocolate with Sprinkles"));
}
}