be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #340 from wb9688/peertube-deleted-comments 

Handle isDeleted for PeerTube comments
This commit is contained in:
Tobias Groza 2020-06-28 10:21:37 +02:00 committed by GitHub
parent f4ed7ce922 3239aa84f2
commit c0ceb5cb27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 96 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -15,20 +15,19 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
public class PeertubeCommentsExtractor extends CommentsExtractor { public class PeertubeCommentsExtractor extends CommentsExtractor {
private InfoItemsPage<CommentsInfoItem> initPage; private InfoItemsPage<CommentsInfoItem> initPage;
private long total; private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler); super(service, uiHandler);
} }
@ -38,22 +37,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
return initPage; return initPage;
} }
private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException {
JsonArray contents; final JsonArray contents = json.getArray("data");
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
for (Object c : contents) { for (final Object c : contents) {
if (c instanceof JsonObject) { if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c; final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); if (!item.getBoolean("isDeleted")) {
collector.commit(extractor); final PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
} }
} }
} }
@Override @Override
@ -63,8 +58,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
} }
@Override @Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException { public InfoItemsPage<CommentsInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl); final Response response = getDownloader().get(pageUrl);
JsonObject json = null; JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) { if (response != null && !Utils.isBlank(response.responseBody())) {
try { try {
@ -74,11 +69,11 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
} }
} }
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if (json != null) { if (json != null) {
Number number = JsonUtils.getNumber(json, "total"); final Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue(); if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl); collectCommentsFrom(collector, json);
} else { } else {
throw new ExtractionException("Unable to get peertube comments info"); throw new ExtractionException("Unable to get peertube comments info");
} }

137
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -18,74 +18,97 @@ import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest { public class PeertubeCommentsExtractorTest {
public static class Default {
private static PeertubeCommentsExtractor extractor;
private static PeertubeCommentsExtractor extractor; @BeforeClass
public static void setUp() throws Exception {
@BeforeClass NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws Exception { extractor = (PeertubeCommentsExtractor) PeerTube
NewPipe.init(DownloaderTestImpl.getInstance()); .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
} }
assertTrue(result); @Test
} public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
@Test boolean result = findInComments(comments, "@root A great documentary on a great guy.");
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { while (comments.hasNextPage() && !result) {
boolean result = false; comments = extractor.getPage(comments.getNextPageUrl());
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); result = findInComments(comments, "@root A great documentary on a great guy.");
assertEquals("Comments", commentsInfo.getName()); }
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl(); assertTrue(result);
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
} }
assertTrue(result); @Test
} public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
assertEquals("Comments", commentsInfo.getName());
@Test boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
public void testGetCommentsAllData() throws IOException, ExtractionException { String nextPage = commentsInfo.getNextPageUrl();
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage(); while (!Utils.isBlank(nextPage) && !result) {
for (CommentsInfoItem c : comments.getItems()) { final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
assertFalse(Utils.isBlank(c.getUploaderUrl())); result = findInComments(moreItems.getItems(), "Loved it!!!");
assertFalse(Utils.isBlank(c.getUploaderName())); nextPage = moreItems.getNextPageUrl();
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); }
assertFalse(Utils.isBlank(c.getCommentId())); assertTrue(result);
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
} }
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) { @Test
return findInComments(comments.getItems(), comment); public void testGetCommentsAllData() throws IOException, ExtractionException {
} final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
private boolean findInComments(List<CommentsInfoItem> comments, String comment) { assertFalse(Utils.isBlank(c.getUploaderUrl()));
for (CommentsInfoItem c : comments) { assertFalse(Utils.isBlank(c.getUploaderName()));
if (c.getCommentText().contains(comment)) { assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
return true; assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
} }
} }
return false;
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}
public static class DeletedComments {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
assertTrue(commentsInfo.getErrors().isEmpty());
}
} }
} }