From 3239aa84f237d2af07a4d586bd282b335c188716 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 31 May 2020 13:33:14 +0200 Subject: [PATCH] Handle isDeleted for PeerTube comments --- .../extractors/PeertubeCommentsExtractor.java | 37 ++--- .../PeertubeCommentsExtractorTest.java | 137 ++++++++++-------- 2 files changed, 96 insertions(+), 78 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index 08f682cf9..5497b76a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -15,20 +15,19 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; public class PeertubeCommentsExtractor extends CommentsExtractor { - private InfoItemsPage initPage; private long total; - public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -38,22 +37,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { return initPage; } - private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { - JsonArray contents; - try { - contents = (JsonArray) JsonUtils.getValue(json, "data"); - } catch (Exception e) { - throw new ParsingException("unable to extract comments info", e); - } + private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException { + final JsonArray contents = json.getArray("data"); - for (Object c : contents) { + for (final Object c : contents) { if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); - collector.commit(extractor); + if (!item.getBoolean("isDeleted")) { + final PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); + collector.commit(extractor); + } } } - } @Override @@ -63,8 +58,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { + final Response response = getDownloader().get(pageUrl); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { @@ -74,11 +69,11 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } } - CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); if (json != null) { - Number number = JsonUtils.getNumber(json, "total"); + final Number number = JsonUtils.getNumber(json, "total"); if (number != null) this.total = number.longValue(); - collectStreamsFrom(collector, json, pageUrl); + collectCommentsFrom(collector, json); } else { throw new ExtractionException("Unable to get peertube comments info"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index 7e4923995..d80416db1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -18,74 +18,97 @@ import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; public class PeertubeCommentsExtractorTest { + public static class Default { + private static PeertubeCommentsExtractor extractor; - private static PeertubeCommentsExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); - } - - @Test - public void testGetComments() throws IOException, ExtractionException { - boolean result = false; - InfoItemsPage comments = extractor.getInitialPage(); - result = findInComments(comments, "@root A great documentary on a great guy."); - - while (comments.hasNextPage() && !result) { - comments = extractor.getPage(comments.getNextPageUrl()); - result = findInComments(comments, "@root A great documentary on a great guy."); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (PeertubeCommentsExtractor) PeerTube + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } - assertTrue(result); - } + @Test + public void testGetComments() throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + assertTrue(comments.getErrors().isEmpty()); - @Test - public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { - boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); - assertEquals("Comments", commentsInfo.getName()); - result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); + boolean result = findInComments(comments, "@root A great documentary on a great guy."); + while (comments.hasNextPage() && !result) { + comments = extractor.getPage(comments.getNextPageUrl()); + result = findInComments(comments, "@root A great documentary on a great guy."); + } - String nextPage = commentsInfo.getNextPageUrl(); - while (!Utils.isBlank(nextPage) && !result) { - InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); - result = findInComments(moreItems.getItems(), "Loved it!!!"); - nextPage = moreItems.getNextPageUrl(); + assertTrue(result); } - assertTrue(result); - } + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + assertTrue(commentsInfo.getErrors().isEmpty()); + assertEquals("Comments", commentsInfo.getName()); - @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { - InfoItemsPage comments = extractor.getInitialPage(); - for (CommentsInfoItem c : comments.getItems()) { - assertFalse(Utils.isBlank(c.getUploaderUrl())); - assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); - assertFalse(Utils.isBlank(c.getCommentId())); - assertFalse(Utils.isBlank(c.getCommentText())); - assertFalse(Utils.isBlank(c.getName())); - assertFalse(Utils.isBlank(c.getTextualUploadDate())); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); - assertFalse(Utils.isBlank(c.getUrl())); - assertFalse(c.getLikeCount() != -1); + boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); + String nextPage = commentsInfo.getNextPageUrl(); + while (!Utils.isBlank(nextPage) && !result) { + final InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); + result = findInComments(moreItems.getItems(), "Loved it!!!"); + nextPage = moreItems.getNextPageUrl(); + } + assertTrue(result); } - } - private boolean findInComments(InfoItemsPage comments, String comment) { - return findInComments(comments.getItems(), comment); - } - - private boolean findInComments(List comments, String comment) { - for (CommentsInfoItem c : comments) { - if (c.getCommentText().contains(comment)) { - return true; + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + final InfoItemsPage comments = extractor.getInitialPage(); + for (final CommentsInfoItem c : comments.getItems()) { + assertFalse(Utils.isBlank(c.getUploaderUrl())); + assertFalse(Utils.isBlank(c.getUploaderName())); + assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + assertFalse(Utils.isBlank(c.getCommentId())); + assertFalse(Utils.isBlank(c.getCommentText())); + assertFalse(Utils.isBlank(c.getName())); + assertFalse(Utils.isBlank(c.getTextualUploadDate())); + assertFalse(Utils.isBlank(c.getThumbnailUrl())); + assertFalse(Utils.isBlank(c.getUrl())); + assertEquals(-1, c.getLikeCount()); } } - return false; + + private boolean findInComments(final InfoItemsPage comments, final String comment) { + return findInComments(comments.getItems(), comment); + } + + private boolean findInComments(final List comments, final String comment) { + for (final CommentsInfoItem c : comments) { + if (c.getCommentText().contains(comment)) { + return true; + } + } + return false; + } + } + + public static class DeletedComments { + private static PeertubeCommentsExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (PeertubeCommentsExtractor) PeerTube + .getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3"); + } + + @Test + public void testGetComments() throws IOException, ExtractionException { + final InfoItemsPage comments = extractor.getInitialPage(); + assertTrue(comments.getErrors().isEmpty()); + } + + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3"); + assertTrue(commentsInfo.getErrors().isEmpty()); + } } }