refactor comments

rename methods in CommentsInfoItemExtractor interface to match the other method names across NewPipeExtractor
remove getName in (Youtube|SoundCloud|Peertube)CommentsExtractor and move it up in CommentsExtractor, return "Comments" instead
This commit is contained in:
bopol 2020-04-21 19:49:17 +02:00
parent 0cbbc2a1f9
commit 2564bcf399
12 changed files with 79 additions and 105 deletions

View File

@ -2,8 +2,11 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem> { public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem> {
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
@ -11,4 +14,9 @@ public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem>
// TODO Auto-generated constructor stub // TODO Auto-generated constructor stub
} }
@Nonnull
@Override
public String getName() throws ParsingException {
return "Comments";
}
} }

View File

@ -9,12 +9,12 @@ public class CommentsInfoItem extends InfoItem {
private String commentId; private String commentId;
private String commentText; private String commentText;
private String authorName; private String uploaderName;
private String authorThumbnail; private String uploaderAvatarUrl;
private String authorEndpoint; private String uploaderUrl;
private String textualPublishedTime; private String textualUploadDate;
@Nullable @Nullable
private DateWrapper publishedTime; private DateWrapper uploadDate;
private int likeCount; private int likeCount;
public CommentsInfoItem(int serviceId, String url, String name) { public CommentsInfoItem(int serviceId, String url, String name) {
@ -37,45 +37,45 @@ public class CommentsInfoItem extends InfoItem {
this.commentText = commentText; this.commentText = commentText;
} }
public String getAuthorName() { public String getUploaderName() {
return authorName; return uploaderName;
} }
public void setAuthorName(String authorName) { public void setUploaderName(String uploaderName) {
this.authorName = authorName; this.uploaderName = uploaderName;
} }
public String getAuthorThumbnail() { public String getUploaderAvatarUrl() {
return authorThumbnail; return uploaderAvatarUrl;
} }
public void setAuthorThumbnail(String authorThumbnail) { public void setUploaderAvatarUrl(String uploaderAvatarUrl) {
this.authorThumbnail = authorThumbnail; this.uploaderAvatarUrl = uploaderAvatarUrl;
} }
public String getAuthorEndpoint() { public String getUploaderUrl() {
return authorEndpoint; return uploaderUrl;
} }
public void setAuthorEndpoint(String authorEndpoint) { public void setUploaderUrl(String uploaderUrl) {
this.authorEndpoint = authorEndpoint; this.uploaderUrl = uploaderUrl;
} }
public String getTextualPublishedTime() { public String getTextualUploadDate() {
return textualPublishedTime; return textualUploadDate;
} }
public void setTextualPublishedTime(String textualPublishedTime) { public void setTextualUploadDate(String textualUploadDate) {
this.textualPublishedTime = textualPublishedTime; this.textualUploadDate = textualUploadDate;
} }
@Nullable @Nullable
public DateWrapper getPublishedTime() { public DateWrapper getUploadDate() {
return publishedTime; return uploadDate;
} }
public void setPublishedTime(@Nullable DateWrapper publishedTime) { public void setUploadDate(@Nullable DateWrapper uploadDate) {
this.publishedTime = publishedTime; this.uploadDate = uploadDate;
} }
public int getLikeCount() { public int getLikeCount() {

View File

@ -9,14 +9,9 @@ import javax.annotation.Nullable;
public interface CommentsInfoItemExtractor extends InfoItemExtractor { public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/**
* AuthorEndpoint, in other words, link to authors' channel page
*/
String getAuthorEndpoint() throws ParsingException;
/** /**
* Return the like count of the comment, or -1 if it's unavailable * Return the like count of the comment, or -1 if it's unavailable
* see {@link StreamExtractor#getLikeCount()} * @see StreamExtractor#getLikeCount()
*/ */
int getLikeCount() throws ParsingException; int getLikeCount() throws ParsingException;
@ -27,20 +22,22 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/** /**
* The upload date given by the service, unmodified * The upload date given by the service, unmodified
* see {@link StreamExtractor#getTextualUploadDate()} * @see StreamExtractor#getTextualUploadDate()
*/ */
String getTextualPublishedTime() throws ParsingException; String getTextualUploadDate() throws ParsingException;
/** /**
* The upload date wrapped with DateWrapper class * The upload date wrapped with DateWrapper class
* see {@link StreamExtractor#getUploadDate()} * @see StreamExtractor#getUploadDate()
*/ */
@Nullable @Nullable
DateWrapper getPublishedTime() throws ParsingException; DateWrapper getUploadDate() throws ParsingException;
String getCommentId() throws ParsingException; String getCommentId() throws ParsingException;
String getAuthorName() throws ParsingException; String getUploaderUrl() throws ParsingException;
String getAuthorThumbnail() throws ParsingException; String getUploaderName() throws ParsingException;
String getUploaderAvatarUrl() throws ParsingException;
} }

View File

@ -35,27 +35,27 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
addError(e); addError(e);
} }
try { try {
resultItem.setAuthorName(extractor.getAuthorName()); resultItem.setUploaderName(extractor.getUploaderName());
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.setAuthorThumbnail(extractor.getAuthorThumbnail()); resultItem.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl());
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.setAuthorEndpoint(extractor.getAuthorEndpoint()); resultItem.setUploaderUrl(extractor.getUploaderUrl());
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.setTextualPublishedTime(extractor.getTextualPublishedTime()); resultItem.setTextualUploadDate(extractor.getTextualUploadDate());
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.setPublishedTime(extractor.getPublishedTime()); resultItem.setUploadDate(extractor.getUploadDate());
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

View File

@ -33,11 +33,6 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
super(service, uiHandler); super(service, uiHandler);
} }
@Override
public String getName() throws ParsingException {
return "Comments";
}
@Override @Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage(); super.fetchPage();

View File

@ -45,13 +45,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
} }
@Override @Override
public String getTextualPublishedTime() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(item, "createdAt"); return JsonUtils.getString(item, "createdAt");
} }
@Override @Override
public DateWrapper getPublishedTime() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
String textualUploadDate = getTextualPublishedTime(); String textualUploadDate = getTextualUploadDate();
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
} }
@ -78,7 +78,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
} }
@Override @Override
public String getAuthorThumbnail() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
String value; String value;
try { try {
value = JsonUtils.getString(item, "account.avatar.path"); value = JsonUtils.getString(item, "account.avatar.path");
@ -89,12 +89,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
} }
@Override @Override
public String getAuthorName() throws ParsingException { public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
} }
@Override @Override
public String getAuthorEndpoint() throws ParsingException { public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name"); String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host"); String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();

View File

@ -67,14 +67,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
} }
} }
@Nonnull
@Override
public String getName() throws ParsingException {
return "SoundCloud comments of track " + getId();
}
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException {
String url = getUrl(); final String url = getUrl();
for (Object comment : entries) { for (Object comment : entries) {
collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url));
} }

View File

@ -29,29 +29,29 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
} }
@Override @Override
public String getAuthorName() throws ParsingException { public String getUploaderName() throws ParsingException {
return json.getObject("user").getString("username"); return json.getObject("user").getString("username");
} }
@Override @Override
public String getAuthorThumbnail() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
return json.getObject("user").getString("avatar_url"); return json.getObject("user").getString("avatar_url");
} }
@Override @Override
public String getAuthorEndpoint() throws ParsingException { public String getUploaderUrl() throws ParsingException {
return json.getObject("user").getString("permalink_url"); return json.getObject("user").getString("permalink_url");
} }
@Override @Override
public String getTextualPublishedTime() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
return json.getString("created_at"); return json.getString("created_at");
} }
@Nullable @Nullable
@Override @Override
public DateWrapper getPublishedTime() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualPublishedTime())); return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
} }
@Override @Override

View File

@ -37,7 +37,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String ytClientVersion; private String ytClientVersion;
private String ytClientName; private String ytClientName;
private String title;
private InfoItemsPage<CommentsInfoItem> initPage; private InfoItemsPage<CommentsInfoItem> initPage;
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
@ -116,7 +115,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
//no comments //no comments
return; return;
} }
fetchTitle(contents);
List<Object> comments; List<Object> comments;
try { try {
comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
@ -132,16 +130,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
} }
} }
private void fetchTitle(JsonArray contents) {
if (title == null) {
try {
title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
} catch (Exception e) {
title = "Youtube Comments";
}
}
}
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final Map<String, List<String>> requestHeaders = new HashMap<>(); final Map<String, List<String>> requestHeaders = new HashMap<>();
@ -155,12 +143,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
initPage = getPage(getNextPageUrl(commentsToken)); initPage = getPage(getNextPageUrl(commentsToken));
} }
@Nonnull
@Override
public String getName() throws ParsingException {
return title;
}
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException { private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
Map<String, List<String>> requestHeaders = new HashMap<>(); Map<String, List<String>> requestHeaders = new HashMap<>();

View File

@ -48,7 +48,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
@Override @Override
public String getTextualPublishedTime() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) { } catch (Exception e) {
@ -58,8 +58,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Nullable @Nullable
@Override @Override
public DateWrapper getPublishedTime() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
String textualPublishedTime = getTextualPublishedTime(); String textualPublishedTime = getTextualUploadDate();
if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) { if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime); return timeAgoParser.parse(textualPublishedTime);
} else { } else {
@ -97,7 +97,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
@Override @Override
public String getAuthorThumbnail() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails"); JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url"); return JsonUtils.getString(arr.getObject(2), "url");
@ -107,7 +107,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
@Override @Override
public String getAuthorName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
@ -116,7 +116,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
@Override @Override
public String getAuthorEndpoint() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId"); return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) { } catch (Exception e) {

View File

@ -14,8 +14,7 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeComment
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.*;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest { public class PeertubeCommentsExtractorTest {
@ -47,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false; boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName())); assertEquals("Comments", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl(); String nextPage = commentsInfo.getNextPageUrl();
@ -64,13 +63,13 @@ public class PeertubeCommentsExtractorTest {
public void testGetCommentsAllData() throws IOException, ExtractionException { public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage(); InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) { for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getUploaderUrl()));
assertFalse(StringUtil.isBlank(c.getAuthorName())); assertFalse(StringUtil.isBlank(c.getUploaderName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl()));
assertFalse(StringUtil.isBlank(c.getCommentId())); assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText())); assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName())); assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); assertFalse(StringUtil.isBlank(c.getTextualUploadDate()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl())); assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1); assertFalse(c.getLikeCount() != -1);

View File

@ -68,15 +68,14 @@ public class YoutubeCommentsExtractorTest {
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
boolean result = false; boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo(url); CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
assertEquals("what the fuck am i doing with my life", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
String nextPage = commentsInfo.getNextPageUrl(); /* String nextPage = commentsInfo.getNextPageUrl();
while (!StringUtil.isBlank(nextPage) && !result) { while (!StringUtil.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3"); result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl(); nextPage = moreItems.getNextPageUrl();
} }*/
return result; return result;
} }
@ -86,14 +85,14 @@ public class YoutubeCommentsExtractorTest {
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) { for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getUploaderUrl()));
assertFalse(StringUtil.isBlank(c.getAuthorName())); assertFalse(StringUtil.isBlank(c.getUploaderName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl()));
assertFalse(StringUtil.isBlank(c.getCommentId())); assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText())); assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName())); assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); assertFalse(StringUtil.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getPublishedTime()); assertNotNull(c.getUploadDate());
assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl())); assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0); assertFalse(c.getLikeCount() < 0);