be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix displaying replies which do not start with the mention of another user

This commit is contained in:
TobiGr 2022-12-04 16:54:45 +01:00
parent e9bbc5dace
commit b6e3015ee2
3 changed files with 82 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -326,14 +326,8 @@ public final class SoundcloudParsingHelper {
public static boolean isReplyTo(@Nonnull final JsonObject originalComment,
@Nonnull final JsonObject otherComment) {
final String mention = "@" + originalComment.getObject("user").getString("permalink");
return otherComment.getString("body").startsWith(mention)
&& originalComment.getInt("timestamp") == otherComment.getInt("timestamp");
return originalComment.getInt("timestamp") == otherComment.getInt("timestamp");
}
public static boolean isReply(@Nonnull final JsonObject comment) {
return comment.getString("body").startsWith("@");
}
}

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
@ -22,10 +24,9 @@ import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor {
public static final String COLLECTION = "collection";
public static final String NEXT_HREF = "next_href";
public SoundcloudCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) {
@ -49,9 +50,9 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json);
collectCommentsFrom(collector, json);
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF)));
}
@Override
@ -83,15 +84,15 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
try {
json = JsonParser.object().from(response.responseBody());
hasNextPage = json.has("next_href");
hasNextPage = json.has(NEXT_HREF);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
collectStreamsFrom(collector, json);
collectCommentsFrom(collector, json);
}
if (hasNextPage) {
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF)));
} else {
return new InfoItemsPage<>(collector, null);
}
@ -100,17 +101,19 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector,
final JsonObject json) throws ParsingException {
private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
final JsonObject json) throws ParsingException {
final String url = getUrl();
final JsonArray entries = json.getArray(COLLECTION);
JsonObject lastTopComment = null;
for (int i = 0; i < entries.size(); i++) {
final JsonObject entry = entries.getObject(i);
if (i == 0
|| (!SoundcloudParsingHelper.isReply(entry)
&& !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry))) {
|| (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry)
&& !SoundcloudParsingHelper.isReplyTo(lastTopComment, entry))) {
lastTopComment = entry;
collector.commit(new SoundcloudCommentsInfoItemExtractor(
json, i, entries.getObject(i), url));
json, i, entry, url));
}
}
}
@ -118,7 +121,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector,
final JsonObject json,
final int id,
final String url) throws ParsingException {
final String url) {
JsonObject originalComment = null;
final JsonArray entries = json.getArray(COLLECTION);
boolean moreReplies = false;
@ -134,7 +137,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
json, i, entries.getObject(i), url, originalComment));
// There might be more replies to the originalComment,
// especially if the original comment is at the end of the list.
if (i == entries.size() - 1 && json.has("next_href")) {
if (i == entries.size() - 1 && json.has(NEXT_HREF)) {
moreReplies = true;
}
}

106
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -1,7 +1,10 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -12,20 +15,21 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
public static final String USER = "user";
public static final String BODY = "body";
public static final String USER_PERMALINK = "permalink";
public static final String USER_FULL_NAME = "full_name";
public static final String USER_USERNAME = "username";
private final JsonObject json;
private final int index;
private final JsonObject item;
private final String url;
private final JsonObject user;
private final JsonObject superComment;
private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT;
@ -39,6 +43,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
this.item = item;
this.url = url;
this.superComment = superComment;
this.user = item.getObject("user");
}
public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index,
@ -50,7 +55,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
public String getCommentId() {
return Objects.toString(item.getLong("id"), null);
}
@Override
public Description getCommentText() {
String commentContent = item.getString(BODY);
@ -61,32 +65,49 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
// Therefore, the comment starts with the mention of the original comment's author.
// The account is automatically linked by the SoundCloud web UI.
// We need to do this manually.
final JsonObject user = superComment.getObject("user");
final String link = "<a href=\"" + user.getString("permalink_url") + "\">"
+ "@" + user.getString("full_name") + "</a>";
commentContent = commentContent
.replace("@" + user.getString(USER_PERMALINK), link)
.replace("@" + superComment.getInt("user_id"), link);
if (commentContent.startsWith("@")) {
final String authorName = commentContent.split(" ", 2)[0].replace("@", "");
final JsonArray comments = json.getArray(SoundcloudCommentsExtractor.COLLECTION);
JsonObject author = null;
for (int i = index - 1; i >= 0 && author == null; i--) {
final JsonObject commentsAuthor = comments.getObject(i).getObject("user");
// use startsWith because sometimes the mention of the user
// is followed by a punctuation character.
if (authorName.startsWith(commentsAuthor.getString(USER_PERMALINK))) {
author = commentsAuthor;
}
}
if (author == null) {
author = superComment.getObject("user");
}
final String name = isNullOrEmpty(author.getString(USER_FULL_NAME))
? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME);
final String link = "<a href=\"" + author.getString("permalink_url") + "\">"
+ "@" + name + "</a>";
commentContent = commentContent
.replace("@" + author.getString(USER_PERMALINK), link)
.replace("@" + author.getInt("user_id"), link);
}
return new Description(commentContent, Description.HTML);
}
@Override
public String getUploaderName() {
if (isNullOrEmpty(user.getString("full_name"))) {
return user.getString("username");
if (isNullOrEmpty(user.getString(USER_FULL_NAME))) {
return user.getString(USER_USERNAME);
}
return user.getString("full_name");
return user.getString(USER_FULL_NAME);
}
@Override
public String getUploaderAvatarUrl() {
return item.getObject(USER).getString("avatar_url");
return user.getString("avatar_url");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return item.getObject(USER).getBoolean("verified");
return user.getBoolean("verified");
}
@Override
@ -96,7 +117,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override
public String getUploaderUrl() {
return item.getObject(USER).getString("permalink_url");
return user.getString("permalink_url");
}
@Override
@ -112,7 +133,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override
public String getName() throws ParsingException {
return item.getObject(USER).getString("permalink");
return user.getString(USER_PERMALINK);
}
@Override
@ -122,38 +143,39 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
@Override
public String getThumbnailUrl() {
return item.getObject(USER).getString("avatar_url");
return user.getString("avatar_url");
}
@Override
public Page getReplies() {
if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) {
final List<JsonObject> replies = new ArrayList<>();
final JsonArray replies = new JsonArray();
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
ServiceList.SoundCloud.getServiceId());
final JsonArray jsonArray = new JsonArray();
// Replies start with the mention of the user who created the original comment.
final String mention = "@" + item.getObject(USER).getString(USER_PERMALINK);
// Loop through all comments which come after the original comment to find its replies.
final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION);
for (int i = index + 1; i < allItems.size(); i++) {
final JsonObject comment = allItems.getObject(i);
final String commentContent = comment.getString("body");
if (commentContent.startsWith(mention)) {
replies.add(comment);
jsonArray.add(comment);
collector.commit(new SoundcloudCommentsInfoItemExtractor(
json, i, comment, url, item));
} else if (!commentContent.startsWith("@") || replies.isEmpty()) {
// Only the comments directly after the original comment
// starting with the mention of the comment's creator
// are replies to the original comment.
// The first comment not starting with these letters
// is the next top-level comment.
break;
// SoundCloud has only comments and top level replies, but not nested replies.
// Therefore, replies cannot have further replies.
if (superComment == null) {
// Loop through all comments which come after the original comment
// to find its replies.
final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION);
boolean foundReply = false;
for (int i = index + 1; i < allItems.size(); i++) {
final JsonObject comment = allItems.getObject(i);
if (SoundcloudParsingHelper.isReplyTo(item, comment)) {
replies.add(comment);
collector.commit(new SoundcloudCommentsInfoItemExtractor(
json, i, comment, url, item));
foundReply = true;
} else if (foundReply) {
// Only the comments directly after the original comment
// having the same timestamp are replies to the original comment.
// The first comment not having the same timestamp
// is the next top-level comment.
break;
}
}
}
replyCount = jsonArray.size();
replyCount = replies.size();
if (collector.getItems().isEmpty()) {
return null;
}
@ -165,7 +187,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
}
@Override
public int getReplyCount() throws ParsingException {
public int getReplyCount() {
if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) {
getReplies();
}