be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #340 from wb9688/peertube-deleted-comments 

Handle isDeleted for PeerTube comments
This commit is contained in:
Tobias Groza 2020-06-28 10:21:37 +02:00 committed by GitHub
parent f4ed7ce922 3239aa84f2
commit c0ceb5cb27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 96 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -15,20 +15,19 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
public class PeertubeCommentsExtractor extends CommentsExtractor {
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@ -38,22 +37,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
return initPage;
}
private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException {
final JsonArray contents = json.getArray("data");
for (Object c : contents) {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
if (!item.getBoolean("isDeleted")) {
final PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
}
}
}
@Override
@ -63,8 +58,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
public InfoItemsPage<CommentsInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
final Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
@ -74,11 +69,11 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
final Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
collectCommentsFrom(collector, json);
} else {
throw new ExtractionException("Unable to get peertube comments info");
}

137
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -18,74 +18,97 @@ import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
public static class Default {
private static PeertubeCommentsExtractor extractor;
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
assertTrue(result);
}
@Test
public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
assertTrue(result);
}
assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
assertEquals("Comments", commentsInfo.getName());
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
}
assertTrue(result);
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
}
}
return false;
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}
public static class DeletedComments {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
assertTrue(commentsInfo.getErrors().isEmpty());
}
}
}