From d4352f9b848ee7e02c9523597399b4b3ccec9dac Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 10 Apr 2020 14:48:11 +0200 Subject: [PATCH] support comments for SoundCloud --- .../extractor/comments/CommentsInfoItem.java | 3 +- .../comments/CommentsInfoItemExtractor.java | 37 +++++++-- .../soundcloud/SoundcloudParsingHelper.java | 6 +- .../soundcloud/SoundcloudService.java | 8 +- .../SoundcloudCommentsExtractor.java | 82 +++++++++++++++++++ .../SoundcloudCommentsInfoItemExtractor.java | 76 +++++++++++++++++ .../extractors/SoundcloudStreamExtractor.java | 2 +- .../SoundcloudStreamInfoItemExtractor.java | 2 +- .../SoundcloudCommentsLinkHandlerFactory.java | 47 +++++++++++ 9 files changed, 250 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java index 6d6290716..30b8c47d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java @@ -13,7 +13,8 @@ public class CommentsInfoItem extends InfoItem { private String authorThumbnail; private String authorEndpoint; private String textualPublishedTime; - @Nullable private DateWrapper publishedTime; + @Nullable + private DateWrapper publishedTime; private int likeCount; public CommentsInfoItem(int serviceId, String url, String name) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 7b40f8fa6..2b9fc3759 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -3,17 +3,44 @@ package org.schabi.newpipe.extractor.comments; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import javax.annotation.Nullable; public interface CommentsInfoItemExtractor extends InfoItemExtractor { - String getCommentId() throws ParsingException; - String getCommentText() throws ParsingException; - String getAuthorName() throws ParsingException; - String getAuthorThumbnail() throws ParsingException; + + /** + * AuthorEndpoint, in other words, link to authors' channel page + */ String getAuthorEndpoint() throws ParsingException; + + /** + * Return the like count of the comment, or -1 if it's unavailable + * see {@link StreamExtractor#getLikeCount()} + */ + int getLikeCount() throws ParsingException; + + /** + * The text of the comment + */ + String getCommentText() throws ParsingException; + + /** + * The upload date given by the service, unmodified + * see {@link StreamExtractor#getTextualUploadDate()} + */ String getTextualPublishedTime() throws ParsingException; + + /** + * The upload date wrapped with DateWrapper class + * see {@link StreamExtractor#getUploadDate()} + */ @Nullable DateWrapper getPublishedTime() throws ParsingException; - int getLikeCount() throws ParsingException; + + String getCommentId() throws ParsingException; + + String getAuthorName() throws ParsingException; + + String getAuthorThumbnail() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 1d15d12b8..da1c9ae6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -88,10 +88,12 @@ public class SoundcloudParsingHelper { } } - public static Calendar parseDate(String textualUploadDate) throws ParsingException { + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(textualUploadDate); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e1) { try { date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 6a6f9ebc3..e9cdb9550 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -17,13 +17,15 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.util.List; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { - super(id, "SoundCloud", singletonList(AUDIO)); + super(id, "SoundCloud", asList(AUDIO, COMMENTS)); } @Override @@ -119,13 +121,13 @@ public class SoundcloudService extends StreamingService { @Override public ListLinkHandlerFactory getCommentsLHFactory() { - return null; + return SoundcloudCommentsLinkHandlerFactory.getInstance(); } @Override public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return null; + return new SoundcloudCommentsExtractor(this, linkHandler); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java new file mode 100644 index 000000000..8bf773470 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -0,0 +1,82 @@ +package org.schabi.newpipe.extractor.services.soundcloud.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class SoundcloudCommentsExtractor extends CommentsExtractor { + + private JsonObject json; + + public SoundcloudCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + super(service, uiHandler); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return json.getString("next_href"); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Downloader dl = NewPipe.getDownloader(); + Response rp = dl.get(pageUrl); + try { + json = JsonParser.object().from(rp.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + try { + json = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return "SoundCloud comments of track " + getId(); + } + + private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { + String url = getUrl(); + for (Object comment : entries) { + collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java new file mode 100644 index 000000000..5ff7db1fb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.soundcloud.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; + +import javax.annotation.Nullable; + +public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private JsonObject json; + private String url; + + public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) { + this.json = json; + this.url = url; + } + + @Override + public String getCommentId() throws ParsingException { + return json.getNumber("id").toString(); + } + + @Override + public String getCommentText() throws ParsingException { + return json.getString("body"); + } + + @Override + public String getAuthorName() throws ParsingException { + return json.getObject("user").getString("username"); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + return json.getObject("user").getString("avatar_url"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + return json.getObject("user").getString("permalink_url"); + } + + @Override + public String getTextualPublishedTime() throws ParsingException { + return json.getString("created_at"); + } + + @Nullable + @Override + public DateWrapper getPublishedTime() throws ParsingException { + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualPublishedTime())); + } + + @Override + public int getLikeCount() throws ParsingException { + return -1; + } + + @Override + public String getName() throws ParsingException { + return json.getObject("user").getString("permalink"); + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return json.getObject("user").getString("avatar_url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index abf394f05..85f0fb5b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -73,7 +73,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at"))); + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at"))); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index e5298a407..5a56bbfb0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -49,7 +49,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate())); + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); } private String getCreatedAt() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java new file mode 100644 index 000000000..f899c9506 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; + +public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory(); + + public static SoundcloudCommentsLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + try { + return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() + + "&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new + // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) + // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) + } catch (ExtractionException | IOException e) { + throw new ParsingException("Could not get comments"); + } + } + + @Override + public String getId(String url) throws ParsingException { + // delagation to avoid duplicate code, as we need the same id + return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); + } + + @Override + public boolean onAcceptUrl(String url) { + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } + } +}