be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix displaying replies which do not start with the mention of another user

This commit is contained in:
TobiGr 2022-12-04 16:54:45 +01:00
parent e9bbc5dace
commit b6e3015ee2
3 changed files with 82 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -326,14 +326,8 @@ public final class SoundcloudParsingHelper {
public static boolean isReplyTo(@Nonnull final JsonObject originalComment, public static boolean isReplyTo(@Nonnull final JsonObject originalComment,
@Nonnull final JsonObject otherComment) { @Nonnull final JsonObject otherComment) {
final String mention = "@" + originalComment.getObject("user").getString("permalink"); return originalComment.getInt("timestamp") == otherComment.getInt("timestamp");
return otherComment.getString("body").startsWith(mention)
&& originalComment.getInt("timestamp") == otherComment.getInt("timestamp");
} }
public static boolean isReply(@Nonnull final JsonObject comment) {
return comment.getString("body").startsWith("@");
}
} }

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors; package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
@ -22,10 +24,9 @@ import java.io.IOException;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor { public class SoundcloudCommentsExtractor extends CommentsExtractor {
public static final String COLLECTION = "collection"; public static final String COLLECTION = "collection";
public static final String NEXT_HREF = "next_href";
public SoundcloudCommentsExtractor(final StreamingService service, public SoundcloudCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) { final ListLinkHandler uiHandler) {
@ -49,9 +50,9 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId()); getServiceId());
collectStreamsFrom(collector, json); collectCommentsFrom(collector, json);
return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF)));
} }
@Override @Override
@ -83,15 +84,15 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
try { try {
json = JsonParser.object().from(response.responseBody()); json = JsonParser.object().from(response.responseBody());
hasNextPage = json.has("next_href"); hasNextPage = json.has(NEXT_HREF);
} catch (final JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e); throw new ParsingException("Could not parse json", e);
} }
collectStreamsFrom(collector, json); collectCommentsFrom(collector, json);
} }
if (hasNextPage) { if (hasNextPage) {
return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF)));
} else { } else {
return new InfoItemsPage<>(collector, null); return new InfoItemsPage<>(collector, null);
} }
@ -100,17 +101,19 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) { } public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
final JsonObject json) throws ParsingException { final JsonObject json) throws ParsingException {
final String url = getUrl(); final String url = getUrl();
final JsonArray entries = json.getArray(COLLECTION); final JsonArray entries = json.getArray(COLLECTION);
JsonObject lastTopComment = null;
for (int i = 0; i < entries.size(); i++) { for (int i = 0; i < entries.size(); i++) {
final JsonObject entry = entries.getObject(i); final JsonObject entry = entries.getObject(i);
if (i == 0 if (i == 0
|| (!SoundcloudParsingHelper.isReply(entry) || (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry)
&& !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry))) { && !SoundcloudParsingHelper.isReplyTo(lastTopComment, entry))) {
lastTopComment = entry;
collector.commit(new SoundcloudCommentsInfoItemExtractor( collector.commit(new SoundcloudCommentsInfoItemExtractor(
json, i, entries.getObject(i), url)); json, i, entry, url));
} }
} }
} }
@ -118,7 +121,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector,
final JsonObject json, final JsonObject json,
final int id, final int id,
final String url) throws ParsingException { final String url) {
JsonObject originalComment = null; JsonObject originalComment = null;
final JsonArray entries = json.getArray(COLLECTION); final JsonArray entries = json.getArray(COLLECTION);
boolean moreReplies = false; boolean moreReplies = false;
@ -134,7 +137,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
json, i, entries.getObject(i), url, originalComment)); json, i, entries.getObject(i), url, originalComment));
// There might be more replies to the originalComment, // There might be more replies to the originalComment,
// especially if the original comment is at the end of the list. // especially if the original comment is at the end of the list.
if (i == entries.size() - 1 && json.has("next_href")) { if (i == entries.size() - 1 && json.has(NEXT_HREF)) {
moreReplies = true; moreReplies = true;
} }
} }

106
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -1,7 +1,10 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors; package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -12,20 +15,21 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import javax.annotation.Nullable;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
public static final String USER = "user";
public static final String BODY = "body"; public static final String BODY = "body";
public static final String USER_PERMALINK = "permalink"; public static final String USER_PERMALINK = "permalink";
public static final String USER_FULL_NAME = "full_name";
public static final String USER_USERNAME = "username";
private final JsonObject json; private final JsonObject json;
private final int index; private final int index;
private final JsonObject item; private final JsonObject item;
private final String url; private final String url;
private final JsonObject user;
private final JsonObject superComment; private final JsonObject superComment;
private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT;
@ -39,6 +43,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
this.item = item; this.item = item;
this.url = url; this.url = url;
this.superComment = superComment; this.superComment = superComment;
this.user = item.getObject("user");
} }
public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index,
@ -50,7 +55,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
public String getCommentId() { public String getCommentId() {
return Objects.toString(item.getLong("id"), null); return Objects.toString(item.getLong("id"), null);
} }
@Override @Override
public Description getCommentText() { public Description getCommentText() {
String commentContent = item.getString(BODY); String commentContent = item.getString(BODY);
@ -61,32 +65,49 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
// Therefore, the comment starts with the mention of the original comment's author. // Therefore, the comment starts with the mention of the original comment's author.
// The account is automatically linked by the SoundCloud web UI. // The account is automatically linked by the SoundCloud web UI.
// We need to do this manually. // We need to do this manually.
final JsonObject user = superComment.getObject("user"); if (commentContent.startsWith("@")) {
final String link = "<a href=\"" + user.getString("permalink_url") + "\">" final String authorName = commentContent.split(" ", 2)[0].replace("@", "");
+ "@" + user.getString("full_name") + "</a>"; final JsonArray comments = json.getArray(SoundcloudCommentsExtractor.COLLECTION);
commentContent = commentContent JsonObject author = null;
.replace("@" + user.getString(USER_PERMALINK), link) for (int i = index - 1; i >= 0 && author == null; i--) {
.replace("@" + superComment.getInt("user_id"), link); final JsonObject commentsAuthor = comments.getObject(i).getObject("user");
// use startsWith because sometimes the mention of the user
// is followed by a punctuation character.
if (authorName.startsWith(commentsAuthor.getString(USER_PERMALINK))) {
author = commentsAuthor;
}
}
if (author == null) {
author = superComment.getObject("user");
}
final String name = isNullOrEmpty(author.getString(USER_FULL_NAME))
? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME);
final String link = "<a href=\"" + author.getString("permalink_url") + "\">"
+ "@" + name + "</a>";
commentContent = commentContent
.replace("@" + author.getString(USER_PERMALINK), link)
.replace("@" + author.getInt("user_id"), link);
}
return new Description(commentContent, Description.HTML); return new Description(commentContent, Description.HTML);
} }
@Override @Override
public String getUploaderName() { public String getUploaderName() {
if (isNullOrEmpty(user.getString("full_name"))) { if (isNullOrEmpty(user.getString(USER_FULL_NAME))) {
return user.getString("username"); return user.getString(USER_USERNAME);
} }
return user.getString("full_name"); return user.getString(USER_FULL_NAME);
} }
@Override @Override
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
return item.getObject(USER).getString("avatar_url"); return user.getString("avatar_url");
} }
@Override @Override
public boolean isUploaderVerified() throws ParsingException { public boolean isUploaderVerified() throws ParsingException {
return item.getObject(USER).getBoolean("verified"); return user.getBoolean("verified");
} }
@Override @Override
@ -96,7 +117,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
return item.getObject(USER).getString("permalink_url"); return user.getString("permalink_url");
} }
@Override @Override
@ -112,7 +133,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return item.getObject(USER).getString("permalink"); return user.getString(USER_PERMALINK);
} }
@Override @Override
@ -122,38 +143,39 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return item.getObject(USER).getString("avatar_url"); return user.getString("avatar_url");
} }
@Override @Override
public Page getReplies() { public Page getReplies() {
if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) {
final List<JsonObject> replies = new ArrayList<>(); final JsonArray replies = new JsonArray();
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
ServiceList.SoundCloud.getServiceId()); ServiceList.SoundCloud.getServiceId());
final JsonArray jsonArray = new JsonArray(); // SoundCloud has only comments and top level replies, but not nested replies.
// Replies start with the mention of the user who created the original comment. // Therefore, replies cannot have further replies.
final String mention = "@" + item.getObject(USER).getString(USER_PERMALINK); if (superComment == null) {
// Loop through all comments which come after the original comment to find its replies. // Loop through all comments which come after the original comment
final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); // to find its replies.
for (int i = index + 1; i < allItems.size(); i++) { final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION);
final JsonObject comment = allItems.getObject(i); boolean foundReply = false;
final String commentContent = comment.getString("body"); for (int i = index + 1; i < allItems.size(); i++) {
if (commentContent.startsWith(mention)) { final JsonObject comment = allItems.getObject(i);
replies.add(comment); if (SoundcloudParsingHelper.isReplyTo(item, comment)) {
jsonArray.add(comment); replies.add(comment);
collector.commit(new SoundcloudCommentsInfoItemExtractor( collector.commit(new SoundcloudCommentsInfoItemExtractor(
json, i, comment, url, item)); json, i, comment, url, item));
} else if (!commentContent.startsWith("@") || replies.isEmpty()) { foundReply = true;
// Only the comments directly after the original comment } else if (foundReply) {
// starting with the mention of the comment's creator // Only the comments directly after the original comment
// are replies to the original comment. // having the same timestamp are replies to the original comment.
// The first comment not starting with these letters // The first comment not having the same timestamp
// is the next top-level comment. // is the next top-level comment.
break; break;
}
} }
} }
replyCount = jsonArray.size(); replyCount = replies.size();
if (collector.getItems().isEmpty()) { if (collector.getItems().isEmpty()) {
return null; return null;
} }
@ -165,7 +187,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
} }
@Override @Override
public int getReplyCount() throws ParsingException { public int getReplyCount() {
if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) {
getReplies(); getReplies();
} }