From 04af575764525b476c4dd883065037b62a8600f8 Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Mon, 30 Mar 2020 00:10:54 +0200 Subject: [PATCH 01/42] Hey Travis, deploy the docs --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5bd81060..137cf3e20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,7 @@ jdk: - openjdk8 script: - - ./gradlew check - - ./gradlew aggregatedJavadocs + - ./gradlew check aggregatedJavadocs deploy: provider: pages From 636c430743c98888edd968ade4950b9971ae4001 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 10 Apr 2020 10:51:05 +0200 Subject: [PATCH 02/42] refactor: create extractors and linkHandler packages for SoundCloud, move YoutubeParsingHelper to youtube package (not linkhandler anymore) --- .../services/soundcloud/SoundcloudParsingHelper.java | 9 ++++++--- .../services/soundcloud/SoundcloudService.java | 2 ++ .../{ => extractors}/SoundcloudChannelExtractor.java | 3 ++- .../SoundcloudChannelInfoItemExtractor.java | 2 +- .../{ => extractors}/SoundcloudChartsExtractor.java | 3 ++- .../{ => extractors}/SoundcloudPlaylistExtractor.java | 3 ++- .../SoundcloudPlaylistInfoItemExtractor.java | 2 +- .../{ => extractors}/SoundcloudSearchExtractor.java | 4 ++-- .../{ => extractors}/SoundcloudStreamExtractor.java | 3 ++- .../SoundcloudStreamInfoItemExtractor.java | 3 ++- .../SoundcloudSubscriptionExtractor.java | 4 +++- .../SoundcloudSuggestionExtractor.java | 3 ++- .../SoundcloudChannelLinkHandlerFactory.java | 3 ++- .../SoundcloudChartsLinkHandlerFactory.java | 2 +- .../SoundcloudPlaylistLinkHandlerFactory.java | 3 ++- .../SoundcloudSearchQueryHandlerFactory.java | 3 ++- .../SoundcloudStreamLinkHandlerFactory.java | 3 ++- .../{linkHandler => }/YoutubeParsingHelper.java | 2 +- .../youtube/extractors/YoutubeChannelExtractor.java | 4 ++-- .../extractors/YoutubeChannelInfoItemExtractor.java | 4 ++-- .../youtube/extractors/YoutubeFeedExtractor.java | 2 +- .../extractors/YoutubeMusicSearchExtractor.java | 10 +++++----- .../youtube/extractors/YoutubePlaylistExtractor.java | 10 +++++----- .../extractors/YoutubePlaylistInfoItemExtractor.java | 4 ++-- .../youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- .../youtube/extractors/YoutubeStreamExtractor.java | 10 +++++----- .../extractors/YoutubeStreamInfoItemExtractor.java | 4 ++-- .../youtube/extractors/YoutubeTrendingExtractor.java | 4 ++-- .../linkHandler/YoutubeChannelLinkHandlerFactory.java | 1 + .../linkHandler/YoutubeCommentsLinkHandlerFactory.java | 2 +- .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 1 + .../linkHandler/YoutubeStreamLinkHandlerFactory.java | 3 ++- .../linkHandler/YoutubeTrendingLinkHandlerFactory.java | 1 + .../peertube/PeertubeTrendingExtractorTest.java | 7 ------- .../soundcloud/SoundcloudChannelExtractorTest.java | 1 + .../soundcloud/SoundcloudChartsExtractorTest.java | 8 +------- .../SoundcloudChartsLinkHandlerFactoryTest.java | 1 + .../soundcloud/SoundcloudPlaylistExtractorTest.java | 1 + .../SoundcloudStreamExtractorDefaultTest.java | 1 + .../SoundcloudStreamLinkHandlerFactoryTest.java | 1 + .../SoundcloudSubscriptionExtractorTest.java | 1 + .../search/SoundcloudSearchExtractorTest.java | 2 +- .../soundcloud/search/SoundcloudSearchQHTest.java | 2 +- .../services/youtube/YoutubeParsingHelperTest.java | 1 - 44 files changed, 80 insertions(+), 67 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudChannelExtractor.java (96%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudChannelInfoItemExtractor.java (94%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudChartsExtractor.java (95%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudPlaylistExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudPlaylistInfoItemExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudSearchExtractor.java (95%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudStreamExtractor.java (98%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudStreamInfoItemExtractor.java (93%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudSubscriptionExtractor.java (92%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => extractors}/SoundcloudSuggestionExtractor.java (92%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => linkHandler}/SoundcloudChannelLinkHandlerFactory.java (91%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => linkHandler}/SoundcloudChartsLinkHandlerFactory.java (93%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => linkHandler}/SoundcloudPlaylistLinkHandlerFactory.java (91%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => linkHandler}/SoundcloudSearchQueryHandlerFactory.java (93%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{ => linkHandler}/SoundcloudStreamLinkHandlerFactory.java (91%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{linkHandler => }/YoutubeParsingHelper.java (99%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index a957ed810..1d15d12b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -15,6 +15,9 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; @@ -85,7 +88,7 @@ public class SoundcloudParsingHelper { } } - static Calendar parseDate(String textualUploadDate) throws ParsingException { + public static Calendar parseDate(String textualUploadDate) throws ParsingException { Date date; try { date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(textualUploadDate); @@ -255,13 +258,13 @@ public class SoundcloudParsingHelper { } @Nonnull - static String getUploaderUrl(JsonObject object) { + public static String getUploaderUrl(JsonObject object) { String url = object.getObject("user").getString("permalink_url", ""); return replaceHttpWithHttps(url); } @Nonnull - static String getAvatarUrl(JsonObject object) { + public static String getAvatarUrl(JsonObject object) { String url = object.getObject("user", new JsonObject()).getString("avatar_url", ""); return replaceHttpWithHttps(url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 1c2998791..6a6f9ebc3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -10,6 +10,8 @@ import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.*; +import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 853b6fc1a..5cadfe15e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java index 118b59cc0..15b9ed84c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java similarity index 95% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index 9a877faba..042af5130 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -1,10 +1,11 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 060b45caf..a529556ee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java index 53123859b..c22766531 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java similarity index 95% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 780e7e2b1..1d05c759d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -22,7 +22,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; public class SoundcloudSearchExtractor extends SearchExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 9ebb3dd6e..abf394f05 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index da0ca1b54..e5298a407 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -1,8 +1,9 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java similarity index 92% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java index cf01b0c93..cabdb453e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java @@ -1,8 +1,10 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java similarity index 92% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java index 924c46983..a204c9951 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java similarity index 91% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java index be1b9fcd3..4ef513b87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java @@ -1,7 +1,8 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java index ef7e700f9..546564e4b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java similarity index 91% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java index c7c882a93..53a70d3aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java @@ -1,7 +1,8 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java index dffcc91f0..ad19acda6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java @@ -1,9 +1,10 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import java.io.IOException; import java.io.UnsupportedEncodingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java similarity index 91% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index fc578411a..e271345d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -1,7 +1,8 @@ -package org.schabi.newpipe.extractor.services.soundcloud; +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java similarity index 99% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 54bfe95c5..0bd3041f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; +package org.schabi.newpipe.extractor.services.youtube; import com.grack.nanojson.JsonArray; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 6a570da53..8ca951fca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -11,7 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -19,7 +19,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.*; /* diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 27c082359..b8c102963 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -7,8 +7,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 12.02.17. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java index 6ed91fe4f..230915f72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java @@ -11,7 +11,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 51918e9ce..ea51f76c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -17,7 +17,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -28,10 +28,10 @@ import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index eef85aa9d..3afc6d39d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -19,10 +19,10 @@ import java.io.IOException; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 5b4e7afc8..5ce475075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -7,8 +7,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private JsonObject playlistInfoItem; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 70c47f6b1..b084ddc83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -17,8 +17,8 @@ import java.io.IOException; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 22.07.2018 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f6b0750c2..ac14f7f10 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -22,7 +22,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; @@ -53,10 +53,10 @@ import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; /* * Created by Christian Schabesberger on 06.08.15. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2330130eb..95ec70c52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -5,7 +5,7 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -16,7 +16,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; /* * Copyright (C) Christian Schabesberger 2016 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index e15463b7f..d1bb95aee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -37,8 +37,8 @@ import java.io.IOException; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 77eaf0694..0eb030852 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.URL; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java index 43baf6a96..15bc31b66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -1,6 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 1950f697b..56abc194b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.URL; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index fd5b5b46d..918e68d8d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -1,11 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.MalformedURLException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 799681128..6d6db7424 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; */ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.MalformedURLException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index e7334fca4..60f72f9d1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -3,17 +3,10 @@ package org.schabi.newpipe.extractor.services.peertube; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudChartsExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.List; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.*; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 485462fbd..1877e11ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index 189be4172..8abfb4d3f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -3,19 +3,13 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.List; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; public class SoundcloudChartsExtractorTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactoryTest.java index 6bcc2e73f..97cc2a80b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactoryTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChartsLinkHandlerFactory; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 59ecff492..6a366fd30 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.hamcrest.CoreMatchers.*; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 75178c4c9..e3fa44b49 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java index 5d303d10d..db2ae2b22 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudStreamLinkHandlerFactory; import java.util.ArrayList; import java.util.List; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java index 507f1ec2e..2bcc1d1ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index 8503f5733..c99bf2a5b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -17,7 +17,7 @@ import java.net.URLEncoder; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.*; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*; public class SoundcloudSearchExtractorTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java index 58a7000d8..2efaa19dc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.*; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*; public class SoundcloudSearchQHTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 669bc3292..0a3e007fc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -5,7 +5,6 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import java.io.IOException; From d4352f9b848ee7e02c9523597399b4b3ccec9dac Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 10 Apr 2020 14:48:11 +0200 Subject: [PATCH 03/42] support comments for SoundCloud --- .../extractor/comments/CommentsInfoItem.java | 3 +- .../comments/CommentsInfoItemExtractor.java | 37 +++++++-- .../soundcloud/SoundcloudParsingHelper.java | 6 +- .../soundcloud/SoundcloudService.java | 8 +- .../SoundcloudCommentsExtractor.java | 82 +++++++++++++++++++ .../SoundcloudCommentsInfoItemExtractor.java | 76 +++++++++++++++++ .../extractors/SoundcloudStreamExtractor.java | 2 +- .../SoundcloudStreamInfoItemExtractor.java | 2 +- .../SoundcloudCommentsLinkHandlerFactory.java | 47 +++++++++++ 9 files changed, 250 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java index 6d6290716..30b8c47d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java @@ -13,7 +13,8 @@ public class CommentsInfoItem extends InfoItem { private String authorThumbnail; private String authorEndpoint; private String textualPublishedTime; - @Nullable private DateWrapper publishedTime; + @Nullable + private DateWrapper publishedTime; private int likeCount; public CommentsInfoItem(int serviceId, String url, String name) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 7b40f8fa6..2b9fc3759 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -3,17 +3,44 @@ package org.schabi.newpipe.extractor.comments; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import javax.annotation.Nullable; public interface CommentsInfoItemExtractor extends InfoItemExtractor { - String getCommentId() throws ParsingException; - String getCommentText() throws ParsingException; - String getAuthorName() throws ParsingException; - String getAuthorThumbnail() throws ParsingException; + + /** + * AuthorEndpoint, in other words, link to authors' channel page + */ String getAuthorEndpoint() throws ParsingException; + + /** + * Return the like count of the comment, or -1 if it's unavailable + * see {@link StreamExtractor#getLikeCount()} + */ + int getLikeCount() throws ParsingException; + + /** + * The text of the comment + */ + String getCommentText() throws ParsingException; + + /** + * The upload date given by the service, unmodified + * see {@link StreamExtractor#getTextualUploadDate()} + */ String getTextualPublishedTime() throws ParsingException; + + /** + * The upload date wrapped with DateWrapper class + * see {@link StreamExtractor#getUploadDate()} + */ @Nullable DateWrapper getPublishedTime() throws ParsingException; - int getLikeCount() throws ParsingException; + + String getCommentId() throws ParsingException; + + String getAuthorName() throws ParsingException; + + String getAuthorThumbnail() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 1d15d12b8..da1c9ae6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -88,10 +88,12 @@ public class SoundcloudParsingHelper { } } - public static Calendar parseDate(String textualUploadDate) throws ParsingException { + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(textualUploadDate); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e1) { try { date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 6a6f9ebc3..e9cdb9550 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -17,13 +17,15 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.util.List; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { - super(id, "SoundCloud", singletonList(AUDIO)); + super(id, "SoundCloud", asList(AUDIO, COMMENTS)); } @Override @@ -119,13 +121,13 @@ public class SoundcloudService extends StreamingService { @Override public ListLinkHandlerFactory getCommentsLHFactory() { - return null; + return SoundcloudCommentsLinkHandlerFactory.getInstance(); } @Override public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return null; + return new SoundcloudCommentsExtractor(this, linkHandler); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java new file mode 100644 index 000000000..8bf773470 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -0,0 +1,82 @@ +package org.schabi.newpipe.extractor.services.soundcloud.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class SoundcloudCommentsExtractor extends CommentsExtractor { + + private JsonObject json; + + public SoundcloudCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + super(service, uiHandler); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return json.getString("next_href"); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Downloader dl = NewPipe.getDownloader(); + Response rp = dl.get(pageUrl); + try { + json = JsonParser.object().from(rp.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + try { + json = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return "SoundCloud comments of track " + getId(); + } + + private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { + String url = getUrl(); + for (Object comment : entries) { + collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java new file mode 100644 index 000000000..5ff7db1fb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.soundcloud.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; + +import javax.annotation.Nullable; + +public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private JsonObject json; + private String url; + + public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) { + this.json = json; + this.url = url; + } + + @Override + public String getCommentId() throws ParsingException { + return json.getNumber("id").toString(); + } + + @Override + public String getCommentText() throws ParsingException { + return json.getString("body"); + } + + @Override + public String getAuthorName() throws ParsingException { + return json.getObject("user").getString("username"); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + return json.getObject("user").getString("avatar_url"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + return json.getObject("user").getString("permalink_url"); + } + + @Override + public String getTextualPublishedTime() throws ParsingException { + return json.getString("created_at"); + } + + @Nullable + @Override + public DateWrapper getPublishedTime() throws ParsingException { + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualPublishedTime())); + } + + @Override + public int getLikeCount() throws ParsingException { + return -1; + } + + @Override + public String getName() throws ParsingException { + return json.getObject("user").getString("permalink"); + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return json.getObject("user").getString("avatar_url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index abf394f05..85f0fb5b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -73,7 +73,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at"))); + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at"))); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index e5298a407..5a56bbfb0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -49,7 +49,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate())); + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); } private String getCreatedAt() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java new file mode 100644 index 000000000..f899c9506 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; + +public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory(); + + public static SoundcloudCommentsLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + try { + return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() + + "&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new + // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) + // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) + } catch (ExtractionException | IOException e) { + throw new ParsingException("Could not get comments"); + } + } + + @Override + public String getId(String url) throws ParsingException { + // delagation to avoid duplicate code, as we need the same id + return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); + } + + @Override + public boolean onAcceptUrl(String url) { + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } + } +} From 0cbbc2a1f96838382b8200a208e8678f5eded77a Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 19 Apr 2020 14:16:38 +0200 Subject: [PATCH 04/42] remove unused import --- .../newpipe/extractor/services/soundcloud/SoundcloudService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index e9cdb9550..e114d030b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -18,7 +18,6 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import java.util.List; import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; From 2564bcf39997cbab8b6e331088c3ee41076e5e00 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 21 Apr 2020 19:49:17 +0200 Subject: [PATCH 05/42] refactor comments rename methods in CommentsInfoItemExtractor interface to match the other method names across NewPipeExtractor remove getName in (Youtube|SoundCloud|Peertube)CommentsExtractor and move it up in CommentsExtractor, return "Comments" instead --- .../extractor/comments/CommentsExtractor.java | 8 +++ .../extractor/comments/CommentsInfoItem.java | 50 +++++++++---------- .../comments/CommentsInfoItemExtractor.java | 21 ++++---- .../comments/CommentsInfoItemsCollector.java | 10 ++-- .../extractors/PeertubeCommentsExtractor.java | 5 -- .../PeertubeCommentsInfoItemExtractor.java | 12 ++--- .../SoundcloudCommentsExtractor.java | 8 +-- .../SoundcloudCommentsInfoItemExtractor.java | 12 ++--- .../extractors/YoutubeCommentsExtractor.java | 18 ------- .../YoutubeCommentsInfoItemExtractor.java | 12 ++--- .../PeertubeCommentsExtractorTest.java | 13 +++-- .../youtube/YoutubeCommentsExtractorTest.java | 15 +++--- 12 files changed, 79 insertions(+), 105 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java index d1599affb..914e631dd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java @@ -2,8 +2,11 @@ package org.schabi.newpipe.extractor.comments; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import javax.annotation.Nonnull; + public abstract class CommentsExtractor extends ListExtractor { public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { @@ -11,4 +14,9 @@ public abstract class CommentsExtractor extends ListExtractor // TODO Auto-generated constructor stub } + @Nonnull + @Override + public String getName() throws ParsingException { + return "Comments"; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java index 30b8c47d9..e8871118a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java @@ -9,12 +9,12 @@ public class CommentsInfoItem extends InfoItem { private String commentId; private String commentText; - private String authorName; - private String authorThumbnail; - private String authorEndpoint; - private String textualPublishedTime; + private String uploaderName; + private String uploaderAvatarUrl; + private String uploaderUrl; + private String textualUploadDate; @Nullable - private DateWrapper publishedTime; + private DateWrapper uploadDate; private int likeCount; public CommentsInfoItem(int serviceId, String url, String name) { @@ -37,45 +37,45 @@ public class CommentsInfoItem extends InfoItem { this.commentText = commentText; } - public String getAuthorName() { - return authorName; + public String getUploaderName() { + return uploaderName; } - public void setAuthorName(String authorName) { - this.authorName = authorName; + public void setUploaderName(String uploaderName) { + this.uploaderName = uploaderName; } - public String getAuthorThumbnail() { - return authorThumbnail; + public String getUploaderAvatarUrl() { + return uploaderAvatarUrl; } - public void setAuthorThumbnail(String authorThumbnail) { - this.authorThumbnail = authorThumbnail; + public void setUploaderAvatarUrl(String uploaderAvatarUrl) { + this.uploaderAvatarUrl = uploaderAvatarUrl; } - public String getAuthorEndpoint() { - return authorEndpoint; + public String getUploaderUrl() { + return uploaderUrl; } - public void setAuthorEndpoint(String authorEndpoint) { - this.authorEndpoint = authorEndpoint; + public void setUploaderUrl(String uploaderUrl) { + this.uploaderUrl = uploaderUrl; } - public String getTextualPublishedTime() { - return textualPublishedTime; + public String getTextualUploadDate() { + return textualUploadDate; } - public void setTextualPublishedTime(String textualPublishedTime) { - this.textualPublishedTime = textualPublishedTime; + public void setTextualUploadDate(String textualUploadDate) { + this.textualUploadDate = textualUploadDate; } @Nullable - public DateWrapper getPublishedTime() { - return publishedTime; + public DateWrapper getUploadDate() { + return uploadDate; } - public void setPublishedTime(@Nullable DateWrapper publishedTime) { - this.publishedTime = publishedTime; + public void setUploadDate(@Nullable DateWrapper uploadDate) { + this.uploadDate = uploadDate; } public int getLikeCount() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 2b9fc3759..cf1501a22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -9,14 +9,9 @@ import javax.annotation.Nullable; public interface CommentsInfoItemExtractor extends InfoItemExtractor { - /** - * AuthorEndpoint, in other words, link to authors' channel page - */ - String getAuthorEndpoint() throws ParsingException; - /** * Return the like count of the comment, or -1 if it's unavailable - * see {@link StreamExtractor#getLikeCount()} + * @see StreamExtractor#getLikeCount() */ int getLikeCount() throws ParsingException; @@ -27,20 +22,22 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor { /** * The upload date given by the service, unmodified - * see {@link StreamExtractor#getTextualUploadDate()} + * @see StreamExtractor#getTextualUploadDate() */ - String getTextualPublishedTime() throws ParsingException; + String getTextualUploadDate() throws ParsingException; /** * The upload date wrapped with DateWrapper class - * see {@link StreamExtractor#getUploadDate()} + * @see StreamExtractor#getUploadDate() */ @Nullable - DateWrapper getPublishedTime() throws ParsingException; + DateWrapper getUploadDate() throws ParsingException; String getCommentId() throws ParsingException; - String getAuthorName() throws ParsingException; + String getUploaderUrl() throws ParsingException; - String getAuthorThumbnail() throws ParsingException; + String getUploaderName() throws ParsingException; + + String getUploaderAvatarUrl() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java index 030d11181..126d7b183 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java @@ -35,27 +35,27 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index b7cbcec7e..166ba69f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -45,13 +45,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac } @Override - public String getTextualPublishedTime() throws ParsingException { + public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(item, "createdAt"); } @Override - public DateWrapper getPublishedTime() throws ParsingException { - String textualUploadDate = getTextualPublishedTime(); + public DateWrapper getUploadDate() throws ParsingException { + String textualUploadDate = getTextualUploadDate(); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } @@ -78,7 +78,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac } @Override - public String getAuthorThumbnail() throws ParsingException { + public String getUploaderAvatarUrl() throws ParsingException { String value; try { value = JsonUtils.getString(item, "account.avatar.path"); @@ -89,12 +89,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac } @Override - public String getAuthorName() throws ParsingException { + public String getUploaderName() throws ParsingException { return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override - public String getAuthorEndpoint() throws ParsingException { + public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index 8bf773470..71c6469d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -67,14 +67,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { } } - @Nonnull - @Override - public String getName() throws ParsingException { - return "SoundCloud comments of track " + getId(); - } - private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { - String url = getUrl(); + final String url = getUrl(); for (Object comment : entries) { collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index 5ff7db1fb..8a478dffd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -29,29 +29,29 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr } @Override - public String getAuthorName() throws ParsingException { + public String getUploaderName() throws ParsingException { return json.getObject("user").getString("username"); } @Override - public String getAuthorThumbnail() throws ParsingException { + public String getUploaderAvatarUrl() throws ParsingException { return json.getObject("user").getString("avatar_url"); } @Override - public String getAuthorEndpoint() throws ParsingException { + public String getUploaderUrl() throws ParsingException { return json.getObject("user").getString("permalink_url"); } @Override - public String getTextualPublishedTime() throws ParsingException { + public String getTextualUploadDate() throws ParsingException { return json.getString("created_at"); } @Nullable @Override - public DateWrapper getPublishedTime() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualPublishedTime())); + public DateWrapper getUploadDate() throws ParsingException { + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 78eb59f3d..5a68501ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -37,7 +37,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { private String ytClientVersion; private String ytClientName; - private String title; private InfoItemsPage initPage; public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { @@ -116,7 +115,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { //no comments return; } - fetchTitle(contents); List comments; try { comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); @@ -132,16 +130,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } } - private void fetchTitle(JsonArray contents) { - if (title == null) { - try { - title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); - } catch (Exception e) { - title = "Youtube Comments"; - } - } - } - @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final Map> requestHeaders = new HashMap<>(); @@ -155,12 +143,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { initPage = getPage(getNextPageUrl(commentsToken)); } - @Nonnull - @Override - public String getName() throws ParsingException { - return title; - } - private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException { Map> requestHeaders = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 4757323ac..b012be7cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -48,7 +48,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } @Override - public String getTextualPublishedTime() throws ParsingException { + public String getTextualUploadDate() throws ParsingException { try { return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText")); } catch (Exception e) { @@ -58,8 +58,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Nullable @Override - public DateWrapper getPublishedTime() throws ParsingException { - String textualPublishedTime = getTextualPublishedTime(); + public DateWrapper getUploadDate() throws ParsingException { + String textualPublishedTime = getTextualUploadDate(); if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) { return timeAgoParser.parse(textualPublishedTime); } else { @@ -97,7 +97,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } @Override - public String getAuthorThumbnail() throws ParsingException { + public String getUploaderAvatarUrl() throws ParsingException { try { JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails"); return JsonUtils.getString(arr.getObject(2), "url"); @@ -107,7 +107,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } @Override - public String getAuthorName() throws ParsingException { + public String getUploaderName() throws ParsingException { try { return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); } catch (Exception e) { @@ -116,7 +116,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } @Override - public String getAuthorEndpoint() throws ParsingException { + public String getUploaderUrl() throws ParsingException { try { return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId"); } catch (Exception e) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index 1241b0a86..ed962b9cd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -14,8 +14,7 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeComment import java.io.IOException; import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; public class PeertubeCommentsExtractorTest { @@ -47,7 +46,7 @@ public class PeertubeCommentsExtractorTest { public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); - assertTrue("Comments".equals(commentsInfo.getName())); + assertEquals("Comments", commentsInfo.getName()); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); String nextPage = commentsInfo.getNextPageUrl(); @@ -64,13 +63,13 @@ public class PeertubeCommentsExtractorTest { public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); for (CommentsInfoItem c : comments.getItems()) { - assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); - assertFalse(StringUtil.isBlank(c.getAuthorName())); - assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getUploaderUrl())); + assertFalse(StringUtil.isBlank(c.getUploaderName())); + assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl())); assertFalse(StringUtil.isBlank(c.getCommentId())); assertFalse(StringUtil.isBlank(c.getCommentText())); assertFalse(StringUtil.isBlank(c.getName())); - assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); + assertFalse(StringUtil.isBlank(c.getTextualUploadDate())); assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); assertFalse(StringUtil.isBlank(c.getUrl())); assertFalse(c.getLikeCount() != -1); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 4349727be..20e49ed14 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -68,15 +68,14 @@ public class YoutubeCommentsExtractorTest { private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { boolean result = false; CommentsInfo commentsInfo = CommentsInfo.getInfo(url); - assertEquals("what the fuck am i doing with my life", commentsInfo.getName()); result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); - String nextPage = commentsInfo.getNextPageUrl(); + /* String nextPage = commentsInfo.getNextPageUrl(); while (!StringUtil.isBlank(nextPage) && !result) { InfoItemsPage moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); result = findInComments(moreItems.getItems(), "s1ck m3m3"); nextPage = moreItems.getNextPageUrl(); - } + }*/ return result; } @@ -86,14 +85,14 @@ public class YoutubeCommentsExtractorTest { DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { - assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); - assertFalse(StringUtil.isBlank(c.getAuthorName())); - assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getUploaderUrl())); + assertFalse(StringUtil.isBlank(c.getUploaderName())); + assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl())); assertFalse(StringUtil.isBlank(c.getCommentId())); assertFalse(StringUtil.isBlank(c.getCommentText())); assertFalse(StringUtil.isBlank(c.getName())); - assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); - assertNotNull(c.getPublishedTime()); + assertFalse(StringUtil.isBlank(c.getTextualUploadDate())); + assertNotNull(c.getUploadDate()); assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); assertFalse(StringUtil.isBlank(c.getUrl())); assertFalse(c.getLikeCount() < 0); From a1eabc75891cdc1c8e5195da488eaeece360e963 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 1 May 2020 13:55:15 +0200 Subject: [PATCH 06/42] Return null instead of "" in getTextFromObject() --- .../extractors/YoutubeChannelExtractor.java | 20 ++++++++----- .../YoutubeMusicSearchExtractor.java | 30 +++++++++++-------- .../extractors/YoutubePlaylistExtractor.java | 6 ++-- .../extractors/YoutubeStreamExtractor.java | 14 +++++---- .../YoutubeStreamInfoItemExtractor.java | 22 ++++++++------ .../extractors/YoutubeTrendingExtractor.java | 6 ++-- .../linkHandler/YoutubeParsingHelper.java | 6 +++- .../MediaCCCStreamExtractorTest.java | 2 +- 8 files changed, 65 insertions(+), 41 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 6a2a4b6c8..8c1b8961f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,11 +17,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; -import static org.schabi.newpipe.extractor.utils.JsonUtils.*; +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Created by Christian Schabesberger on 25.07.16. @@ -306,10 +310,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ContentNotSupportedException("This channel has no Videos tab"); } - if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("messageRenderer") - .getObject("text")).equals("This channel has no videos.")) { + final String messageRendererText = getTextFromObject(videoTab.getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("messageRenderer").getObject("text")); + if (messageRendererText != null + && messageRendererText.equals("This channel has no videos.")) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 5f828a051..4f7847b2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -19,13 +20,14 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -249,7 +251,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get name"); @@ -259,7 +261,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public long getDuration() throws ParsingException { final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!duration.isEmpty()) { + if (duration != null && !duration.isEmpty()) { return YoutubeParsingHelper.parseDurationString(duration); } throw new ParsingException("Could not get duration"); @@ -269,7 +271,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getUploaderName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get uploader name"); @@ -288,11 +290,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return null; } else { - final JsonObject navigationEndpoint = info.getArray("flexColumns") + final JsonObject navigationEndpointHolder = info.getArray("flexColumns") .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + .getObject("text").getArray("runs").getObject(0); - final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + if (!navigationEndpointHolder.has("navigationEndpoint")) return null; + + final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint")); if (url != null && !url.isEmpty()) { return url; @@ -319,7 +323,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!viewCount.isEmpty()) { + if (viewCount != null && !viewCount.isEmpty()) { return Utils.mixedNumberWordToLong(viewCount); } throw new ParsingException("Could not get view count"); @@ -359,7 +363,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get name"); @@ -378,7 +382,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public long getSubscriberCount() throws ParsingException { final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!viewCount.isEmpty()) { + if (viewCount != null && !viewCount.isEmpty()) { return Utils.mixedNumberWordToLong(viewCount); } throw new ParsingException("Could not get subscriber count"); @@ -414,7 +418,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get name"); @@ -439,7 +443,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { name = getTextFromObject(info.getArray("flexColumns").getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); } - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get uploader name"); @@ -452,7 +456,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (!count.isEmpty()) { + if (count != null && !count.isEmpty()) { if (count.contains("100+")) { return ITEM_COUNT_MORE_THAN_100; } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 4f1884193..68b53855a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -14,9 +15,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -81,7 +83,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(playlistInfo.getObject("title")); - if (!name.isEmpty()) return name; + if (name != null && !name.isEmpty()) return name; return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 55268ed73..100e6dcaa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; + import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -35,8 +36,6 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; @@ -50,6 +49,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -117,7 +119,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - if (title.isEmpty()) { + if (title == null || title.isEmpty()) { title = playerResponse.getObject("videoDetails").getString("title"); if (title == null || title.isEmpty()) throw new ParsingException("Could not get name"); @@ -197,7 +199,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // description with more info on links String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); - if (!description.isEmpty()) return new Description(description, Description.HTML); + if (description != null && !description.isEmpty()) return new Description(description, Description.HTML); // raw non-html description return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); @@ -249,7 +251,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount")); - if (views.isEmpty()) { + if (views == null || views.isEmpty()) { views = playerResponse.getObject("videoDetails").getString("viewCount"); if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count"); @@ -330,7 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - if (uploaderName.isEmpty()) { + if (uploaderName == null || uploaderName.isEmpty()) { uploaderName = playerResponse.getObject("videoDetails").getString("author"); if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 787027692..77850dd82 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -11,12 +12,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nullable; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* @@ -93,7 +97,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(videoInfo.getObject("title")); - if (!name.isEmpty()) return name; + if (name != null && !name.isEmpty()) return name; throw new ParsingException("Could not get name"); } @@ -105,7 +109,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { String duration = getTextFromObject(videoInfo.getObject("lengthText")); - if (duration.isEmpty()) { + if (duration == null || duration.isEmpty()) { for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) { duration = getTextFromObject(((JsonObject) thumbnailOverlay) @@ -113,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } - if (duration.isEmpty()) throw new ParsingException("Could not get duration"); + if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -123,13 +127,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { public String getUploaderName() throws ParsingException { String name = getTextFromObject(videoInfo.getObject("longBylineText")); - if (name.isEmpty()) { + if (name == null || name.isEmpty()) { name = getTextFromObject(videoInfo.getObject("ownerText")); - if (name.isEmpty()) { + if (name == null || name.isEmpty()) { name = getTextFromObject(videoInfo.getObject("shortBylineText")); - if (name.isEmpty()) throw new ParsingException("Could not get uploader name"); + if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name"); } } @@ -169,7 +173,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); - if (!publishedTimeText.isEmpty()) return publishedTimeText; + if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText; return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 3a015ba9c..958af8b38 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -22,6 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -32,9 +33,10 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -71,7 +73,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); - if (!name.isEmpty()) { + if (name != null && !name.isEmpty()) { return name; } throw new ParsingException("Could not get Trending name"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 6ebb2497f..250d22d65 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -388,11 +388,15 @@ public class YoutubeParsingHelper { * Get the text from a JSON object that has either a simpleText or a runs array. * @param textObject JSON object to get the text from * @param html whether to return HTML, by parsing the navigationEndpoint - * @return text in the JSON object or an empty string + * @return text in the JSON object or {@code null} */ public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { + if (textObject == null || textObject.isEmpty()) return null; + if (textObject.has("simpleText")) return textObject.getString("simpleText"); + if (textObject.getArray("runs").isEmpty()) return null; + StringBuilder textBuilder = new StringBuilder(); for (Object textPart : textObject.getArray("runs")) { String text = ((JsonObject) textPart).getString("text"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index f376b9f4d..95c882290 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -76,7 +76,7 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); } @Test From 6aae99f832774ab6f6d6d2364dbf7b33efdf92cc Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 3 May 2020 10:28:45 +0200 Subject: [PATCH 07/42] Fix NPE in defaultAlertsCheck() --- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 250d22d65..da0decfab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -28,6 +28,7 @@ import java.text.SimpleDateFormat; import java.util.*; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -494,12 +495,12 @@ public class YoutubeParsingHelper { * @param initialData the object which will be checked if an alert is present * @throws ContentNotAvailableException if an alert is detected */ - public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { + public static void defaultAlertsCheck(final JsonObject initialData) throws ParsingException { final JsonArray alerts = initialData.getArray("alerts"); if (!alerts.isEmpty()) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); - final String alertText = alertRenderer.getObject("text").getString("simpleText"); - final String alertType = alertRenderer.getString("type"); + final String alertText = getTextFromObject(alertRenderer.getObject("text")); + final String alertType = alertRenderer.getString("type", EMPTY_STRING); if (alertType.equalsIgnoreCase("ERROR")) { throw new ContentNotAvailableException("Got error: \"" + alertText + "\""); } From 8a79b14077c59d17b2aea7cffe93feb3ce6f2ed4 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 3 May 2020 15:11:20 +0200 Subject: [PATCH 08/42] Update extractor version to 0.19.4 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6f349545f..611e10332 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.19.0' + version 'v0.19.4' group 'com.github.TeamNewPipe' repositories { From a02bfbeae5b7113cff2069e3823d42db21640fc0 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 6 May 2020 15:41:01 +0200 Subject: [PATCH 09/42] [YouTube] Support stream urls in `vnd.youtube://ID` form --- .../YoutubeStreamLinkHandlerFactory.java | 45 ++++++++++++------- .../YoutubeStreamLinkHandlerFactoryTest.java | 1 + 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index fd5b5b46d..45edfdda2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -13,6 +13,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import javax.annotation.Nullable; + /* * Created by Christian Schabesberger on 02.02.16. * @@ -44,12 +46,16 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return instance; } - private static String assertIsID(String id) throws ParsingException { - if (id == null || !id.matches("[a-zA-Z0-9_-]{11}")) { + private static boolean isId(@Nullable String id) { + return id != null && id.matches("[a-zA-Z0-9_-]{11}"); + } + + private static String assertIsId(@Nullable String id) throws ParsingException { + if (isId(id)) { + return id; + } else { throw new ParsingException("The given string is not a Youtube-Video-ID"); } - - return id; } @Override @@ -75,9 +81,14 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { String schemeSpecificPart = uri.getSchemeSpecificPart(); if (schemeSpecificPart.startsWith("//")) { + final String possiblyId = schemeSpecificPart.substring(2); + if (isId(possiblyId)) { + return possiblyId; + } + urlString = "https:" + schemeSpecificPart; } else { - return assertIsID(schemeSpecificPart); + return assertIsId(schemeSpecificPart); } } } catch (URISyntaxException ignored) { @@ -118,7 +129,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { if (path.startsWith("embed/")) { String id = path.split("/")[1]; - return assertIsID(id); + return assertIsId(id); } break; @@ -139,38 +150,38 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); - return assertIsID(viewQueryValue); + return assertIsId(viewQueryValue); } if (path.startsWith("embed/")) { String id = path.split("/")[1]; - return assertIsID(id); + return assertIsId(id); } String viewQueryValue = Utils.getQueryValue(url, "v"); - return assertIsID(viewQueryValue); + return assertIsId(viewQueryValue); } case "YOUTU.BE": { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { - return assertIsID(viewQueryValue); + return assertIsId(viewQueryValue); } - return assertIsID(path); + return assertIsId(path); } case "HOOKTUBE.COM": { if (path.startsWith("v/")) { String id = path.substring("v/".length()); - return assertIsID(id); + return assertIsId(id); } if (path.startsWith("watch/")) { String id = path.substring("watch/".length()); - return assertIsID(id); + return assertIsId(id); } // there is no break-statement here on purpose so the next code-block gets also run for hooktube } @@ -193,21 +204,21 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { - return assertIsID(viewQueryValue); + return assertIsId(viewQueryValue); } } if (path.startsWith("embed/")) { String id = path.substring("embed/".length()); - return assertIsID(id); + return assertIsId(id); } String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { - return assertIsID(viewQueryValue); + return assertIsId(viewQueryValue); } - return assertIsID(path); + return assertIsId(path); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 908642fdb..62398d597 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -79,6 +79,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); + assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); } From 29b639b454ce38f29d9dcc4c9fe054d581005bbe Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 11 Apr 2020 19:18:08 +0200 Subject: [PATCH 10/42] Search: add isCorrectedSearch() and fix YoutubeSearchExtractor#getSearchSuggestion() isCorrectedSearch: see the javadoc getSearchSuggestion: in YoutubeSearchExtractor, it was giving the corrected search query. It now gives the suggested query, as it should --- .../extractor/search/SearchExtractor.java | 11 ++++++++ .../newpipe/extractor/search/SearchInfo.java | 10 +++++++ .../extractors/MediaCCCSearchExtractor.java | 5 ++++ .../extractors/PeertubeSearchExtractor.java | 5 ++++ .../extractors/SoundcloudSearchExtractor.java | 5 ++++ .../YoutubeMusicSearchExtractor.java | 13 +++++++++ .../extractors/YoutubeSearchExtractor.java | 23 ++++++++++----- .../services/BaseSearchExtractorTest.java | 1 + .../services/DefaultSearchExtractorTest.java | 9 ++++++ .../YoutubeMusicSearchExtractorTest.java | 23 +++++++++++++++ .../search/YoutubeSearchExtractorTest.java | 28 +++++++++++++++++-- 11 files changed, 123 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index a7efea962..0101b96f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -37,4 +37,15 @@ public abstract class SearchExtractor extends ListExtractor { public String getName() { return getLinkHandler().getSearchString(); } + + /** + * When you search on some service, it can give you another and corrected request. + * This method says if it's the case. + *

+ * Example: on YouTube, if you search for "pewdeipie", + * it will give you results for "pewdiepie", then isCorrectedSearch should return true. + * + * @return whether the results comes from a corrected query or not. + */ + public abstract boolean isCorrectedSearch() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index eedce719e..547162bd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo { private String searchString; private String searchSuggestion; + private boolean isCorrectedSearch; public SearchInfo(int serviceId, SearchQueryHandler qIHandler, @@ -46,6 +47,11 @@ public class SearchInfo extends ListInfo { } catch (Exception e) { info.addError(e); } + try { + info.isCorrectedSearch = extractor.isCorrectedSearch(); + } catch (Exception e) { + info.addError(e); + } ListExtractor.InfoItemsPage page = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(page.getItems()); @@ -70,4 +76,8 @@ public class SearchInfo extends ListInfo { public String getSearchSuggestion() { return searchSuggestion; } + + public boolean isCorrectedSearch() { + return this.isCorrectedSearch; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index d5ced534b..67f19a838 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,6 +47,11 @@ public class MediaCCCSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 61fcdd6df..958f53619 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -40,6 +40,11 @@ public class PeertubeSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 92730ec46..b121d56c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -38,6 +38,11 @@ public class SoundcloudSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7e952c94b..99f6ecd2a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -135,6 +135,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } + @Override + public boolean isCorrectedSearch() { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return false; + } + + JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return showingResultsForRenderer != null; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException, IOException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 732ef09ad..69e27e2f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; - -import java.io.IOException; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -64,15 +63,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { + final JsonObject didYouMeanRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } + + @Override + public boolean isCorrectedSearch() { final JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!showingResultsForRenderer.has("correctedQuery")) { - return ""; - } - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + return showingResultsForRenderer != null; } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java index e82ad7d0a..ddfa27fb0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services; public interface BaseSearchExtractorTest extends BaseListExtractorTest { void testSearchString() throws Exception; void testSearchSuggestion() throws Exception; + void testSearchCorrected() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 93c4eac70..b363031d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -15,6 +15,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes public abstract String expectedSearchString(); @Nullable public abstract String expectedSearchSuggestion(); + public boolean isCorrectedSearch() { + return false; + } + @Test @Override public void testSearchString() throws Exception { @@ -31,4 +35,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); } } + + @Test + public void testSearchCorrected() throws Exception { + assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 420db0adb..2dc4c2814 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -150,4 +150,27 @@ public class YoutubeMusicSearchExtractorTest { @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "duo lipa"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d5f2f1af2..9cc4ad1ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -114,8 +114,8 @@ public class YoutubeSearchExtractorTest { public static class Suggestion extends DefaultSearchExtractorTest { private static SearchExtractor extractor; - private static final String QUERY = "pewdeipie"; - private static final String EXPECTED_SUGGESTION = "pewdiepie"; + private static final String QUERY = "newpip"; + private static final String EXPECTED_SUGGESTION = "newpipe"; @BeforeClass public static void setUp() throws Exception { @@ -132,10 +132,32 @@ public class YoutubeSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "pewdeipie"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } + public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "UCO6AK"; From 5d1c3b3fa2f9a28e6caf91621b3a88eff0e81a3c Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 12 Apr 2020 15:27:51 +0200 Subject: [PATCH 11/42] Use setters in SearchInfo --- .../schabi/newpipe/extractor/search/SearchInfo.java | 12 ++++++++++-- .../extractors/YoutubeMusicSearchExtractor.java | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 547162bd1..e5c054571 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -43,12 +43,12 @@ public class SearchInfo extends ListInfo { info.addError(e); } try { - info.searchSuggestion = extractor.getSearchSuggestion(); + info.setSearchSuggestion(extractor.getSearchSuggestion()); } catch (Exception e) { info.addError(e); } try { - info.isCorrectedSearch = extractor.isCorrectedSearch(); + info.setCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -80,4 +80,12 @@ public class SearchInfo extends ListInfo { public boolean isCorrectedSearch() { return this.isCorrectedSearch; } + + public void setCorrectedSearch(boolean correctedSearch) { + isCorrectedSearch = correctedSearch; + } + + public void setSearchSuggestion(String searchSuggestion) { + this.searchSuggestion = searchSuggestion; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 99f6ecd2a..a6ea6531f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -143,7 +143,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return false; } - JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); return showingResultsForRenderer != null; } From e8d58e09c9be249bfa430357c059443cebda6989 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 12 Apr 2020 16:06:52 +0200 Subject: [PATCH 12/42] getSearchSuggestion now returns the corrected query if isCorrectedSearch() is true --- .../extractor/search/SearchExtractor.java | 10 ++++++++ .../extractors/MediaCCCSearchExtractor.java | 6 +++-- .../extractors/PeertubeSearchExtractor.java | 4 +++- .../extractors/SoundcloudSearchExtractor.java | 3 ++- .../YoutubeMusicSearchExtractor.java | 23 +++++++++++++++---- .../extractors/YoutubeSearchExtractor.java | 16 +++++++++---- .../YoutubeMusicSearchExtractorTest.java | 3 ++- .../search/YoutubeSearchExtractorTest.java | 4 +++- 8 files changed, 54 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 0101b96f0..4290d6948 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -25,6 +25,16 @@ public abstract class SearchExtractor extends ListExtractor { return getLinkHandler().getSearchString(); } + /** + * The search suggestion provided by the service. + *

+ * This method may also return the corrected query, + * see {@link SearchExtractor#isCorrectedSearch()}. + * + * @return a suggestion to another query, the corrected query, or an empty String. + * @throws ParsingException + */ + @Nonnull public abstract String getSearchSuggestion() throws ParsingException; @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 67f19a838..c4b368923 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -42,9 +43,10 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } } + @Nonnull @Override - public String getSearchSuggestion() { - return null; + public String getSearchSuggestion() throws ParsingException { + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 958f53619..352164de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import javax.annotation.Nonnull; import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -35,9 +36,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - return null; + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index b121d56c3..24f5987ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -33,9 +33,10 @@ public class SoundcloudSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index a6ea6531f..95b8c9f85 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -124,15 +125,27 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return super.getUrl(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); - if (!didYouMeanRenderer.has("correctedQuery")) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return ""; + } + + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (didYouMeanRenderer != null) { + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } else if (showingResultsForRenderer != null) { + return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else { return ""; } - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 69e27e2f5..8c18c6024 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -61,17 +61,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents") + final JsonObject itemSectionRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer"); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) { + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (didYouMeanRenderer != null) { + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else if (showingResultsForRenderer != null) { + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + } else { return ""; } - return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 2dc4c2814..eeac8c49b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -154,6 +154,7 @@ public class YoutubeMusicSearchExtractorTest { public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "duo lipa"; + private static final String EXPECTED_SUGGESTION = "dua lipa"; @BeforeClass public static void setUp() throws Exception { @@ -169,7 +170,7 @@ public class YoutubeMusicSearchExtractorTest { @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } @Override public String expectedSearchString() { return QUERY; } - @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Override public boolean isCorrectedSearch() { return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 9cc4ad1ff..4f7d46d52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; @@ -138,6 +139,7 @@ public class YoutubeSearchExtractorTest { public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "pewdeipie"; + private static final String EXPECTED_SUGGESTION = "pewdiepie"; @BeforeClass public static void setUp() throws Exception { @@ -153,7 +155,7 @@ public class YoutubeSearchExtractorTest { @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } - @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Override public boolean isCorrectedSearch() { return true; } } From 79483a6dc04a1d37e1e9e661b113632ddb420bf7 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 15 Apr 2020 21:05:31 +0200 Subject: [PATCH 13/42] =?UTF-8?q?=C2=ABUse=20this=20in=20SearchInfo,=20for?= =?UTF-8?q?=20consistency's=20sake=C2=BB=20@Stypox=20order=20xD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/schabi/newpipe/extractor/search/SearchInfo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index e5c054571..9c6dd54de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -70,11 +70,11 @@ public class SearchInfo extends ListInfo { // Getter public String getSearchString() { - return searchString; + return this.searchString; } public String getSearchSuggestion() { - return searchSuggestion; + return this.searchSuggestion; } public boolean isCorrectedSearch() { @@ -82,7 +82,7 @@ public class SearchInfo extends ListInfo { } public void setCorrectedSearch(boolean correctedSearch) { - isCorrectedSearch = correctedSearch; + this.isCorrectedSearch = correctedSearch; } public void setSearchSuggestion(String searchSuggestion) { From 3757541a9c8223d4ffce261faddee95112f125b5 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 5 May 2020 10:02:56 +0200 Subject: [PATCH 14/42] adapt new nanojson version --- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 2 +- .../extractors/YoutubeMusicSearchExtractor.java | 10 +++++----- .../youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- .../youtube/search/YoutubeSearchExtractorTest.java | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index c4b368923..914c77497 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -45,7 +45,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override - public String getSearchSuggestion() throws ParsingException { + public String getSearchSuggestion() { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 95b8c9f85..0731f3041 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -130,7 +130,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) { + if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return ""; } @@ -139,9 +139,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (didYouMeanRenderer != null) { + if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else if (showingResultsForRenderer != null) { + } else if (!showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else { return ""; @@ -152,13 +152,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public boolean isCorrectedSearch() { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) { + if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return false; } final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return showingResultsForRenderer != null; + return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 8c18c6024..f35bc6c06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -73,7 +73,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (didYouMeanRenderer != null) { + if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else if (showingResultsForRenderer != null) { return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); @@ -89,7 +89,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return showingResultsForRenderer != null; + return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 4f7d46d52..84e1501b8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; From 93e868634e1f3ab0fbfaf3724555c3c496108c21 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 5 May 2020 10:09:35 +0200 Subject: [PATCH 15/42] clarified javadoc for isCorrectedSearch and getSearchSuggestions in SearchExtractor --- .../schabi/newpipe/extractor/search/SearchExtractor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 4290d6948..d1c481bb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -28,8 +28,8 @@ public abstract class SearchExtractor extends ListExtractor { /** * The search suggestion provided by the service. *

- * This method may also return the corrected query, - * see {@link SearchExtractor#isCorrectedSearch()}. + * This method also returns the corrected query if + * {@link SearchExtractor#isCorrectedSearch()} is true. * * @return a suggestion to another query, the corrected query, or an empty String. * @throws ParsingException @@ -49,8 +49,7 @@ public abstract class SearchExtractor extends ListExtractor { } /** - * When you search on some service, it can give you another and corrected request. - * This method says if it's the case. + * Tell if the search was corrected by the service (if it's not exactly the search you typed). *

* Example: on YouTube, if you search for "pewdeipie", * it will give you results for "pewdiepie", then isCorrectedSearch should return true. From 5760366d70ca5c6413f759af5b9be2c7e36c919a Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 7 May 2020 15:32:25 +0200 Subject: [PATCH 16/42] use JsonObject#isEmpty instead of JsonObject#equals(JsonUtils.EMPTY_OBJECT) --- .../extractors/YoutubeMusicSearchExtractor.java | 10 +++++----- .../youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 0731f3041..39a6e94d4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -130,7 +130,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (itemSectionRenderer.isEmpty()) { return ""; } @@ -139,9 +139,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (!didYouMeanRenderer.isEmpty()) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else if (!showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + } else if (!showingResultsForRenderer.isEmpty()) { return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else { return ""; @@ -152,13 +152,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public boolean isCorrectedSearch() { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (itemSectionRenderer.isEmpty()) { return false; } final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index f35bc6c06..05c27d53d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -73,7 +73,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (!didYouMeanRenderer.isEmpty()) { return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else if (showingResultsForRenderer != null) { return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); @@ -89,7 +89,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); + return !showingResultsForRenderer.isEmpty(); } @Nonnull From 6cff5de400f33f9e93c79efbf60cc0b7011b3027 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 7 May 2020 15:33:58 +0200 Subject: [PATCH 17/42] renaming methods for consistency --- .../org/schabi/newpipe/extractor/search/SearchInfo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 9c6dd54de..b2e072cce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -48,7 +48,7 @@ public class SearchInfo extends ListInfo { info.addError(e); } try { - info.setCorrectedSearch(extractor.isCorrectedSearch()); + info.setIsCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -81,8 +81,8 @@ public class SearchInfo extends ListInfo { return this.isCorrectedSearch; } - public void setCorrectedSearch(boolean correctedSearch) { - this.isCorrectedSearch = correctedSearch; + public void setIsCorrectedSearch(boolean isCorrectedSearch) { + this.isCorrectedSearch = isCorrectedSearch; } public void setSearchSuggestion(String searchSuggestion) { From 4234740baa6c3df4634620bc047cc482cac1441a Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Mon, 13 Apr 2020 23:33:44 +0300 Subject: [PATCH 18/42] Add parent channel info to ChannelExtractor: name, url & avatar url --- .../extractor/channel/ChannelExtractor.java | 3 ++ .../extractor/channel/ChannelInfo.java | 45 +++++++++++++++++++ .../MediaCCCConferenceExtractor.java | 15 +++++++ .../extractors/PeertubeChannelExtractor.java | 21 +++++++++ .../SoundcloudChannelExtractor.java | 15 +++++++ .../extractors/YoutubeChannelExtractor.java | 15 +++++++ .../PeertubeChannelExtractorTest.java | 20 +++++++++ 7 files changed, 134 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 6d529d6b6..3f54f1c2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -37,4 +37,7 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; + public abstract String getParentChannelName() throws ParsingException; + public abstract String getParentChannelUrl() throws ParsingException; + public abstract String getParentChannelAvatarUrl() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 0fc44063b..52f18eadc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -94,16 +94,61 @@ public class ChannelInfo extends ListInfo { info.addError(e); } + try { + info.setParentChannelName(extractor.getParentChannelName()); + } catch (Exception e) { + info.addError(e); + } + + try { + info.setParentChannelUrl(extractor.getParentChannelUrl()); + } catch (Exception e) { + info.addError(e); + } + + try { + info.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + } catch (Exception e) { + info.addError(e); + } + return info; } private String avatarUrl; + private String parentChannelName; + private String parentChannelUrl; + private String parentChannelAvatarUrl; private String bannerUrl; private String feedUrl; private long subscriberCount = -1; private String description; private String[] donationLinks; + public String getParentChannelName() { + return parentChannelName; + } + + public void setParentChannelName(String parentChannelName) { + this.parentChannelName = parentChannelName; + } + + public String getParentChannelUrl() { + return parentChannelUrl; + } + + public void setParentChannelUrl(String parentChannelUrl) { + this.parentChannelUrl = parentChannelUrl; + } + + public String getParentChannelAvatarUrl() { + return parentChannelAvatarUrl; + } + + public void setParentChannelAvatarUrl(String parentChannelAvatarUrl) { + this.parentChannelAvatarUrl = parentChannelAvatarUrl; + } + public String getAvatarUrl() { return avatarUrl; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 4cd21c060..f11c13c22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -52,6 +52,21 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { return null; } + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index dc27be80c..c8f7f4cb3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -75,6 +75,27 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } } + @Override + public String getParentChannelName() throws ParsingException { + return JsonUtils.getString(json, "ownerAccount.name"); + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return JsonUtils.getString(json, "ownerAccount.url"); + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "ownerAccount.avatar.path"); + } catch (Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index a1d258b54..1068b36f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -83,6 +83,21 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return user.getString("description", EMPTY_STRING); } + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 38d57f95c..0e5329425 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -212,6 +212,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } } + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 9dc4b013e..bdab43ef8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -84,6 +84,16 @@ public class PeertubeChannelExtractorTest { assertNotNull(extractor.getDescription()); } + @Test + public void testParentChannelName() throws ParsingException { + assertEquals("libux", extractor.getParentChannelName()); + } + + @Test + public void testParentChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getParentChannelUrl()); + } + @Test public void testAvatarUrl() throws ParsingException { assertIsSecureUrl(extractor.getAvatarUrl()); @@ -181,6 +191,16 @@ public class PeertubeChannelExtractorTest { assertNotNull(extractor.getDescription()); } + @Test + public void testParentChannelName() throws ParsingException { + assertEquals("booteille", extractor.getParentChannelName()); + } + + @Test + public void testParentChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getParentChannelUrl()); + } + @Test public void testAvatarUrl() throws ParsingException { assertIsSecureUrl(extractor.getAvatarUrl()); From 1de1f97cf1b3a269cba4137d23140df60c334d88 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Mon, 13 Apr 2020 23:34:20 +0300 Subject: [PATCH 19/42] Add parent channel info to StreamExtractor: name, url & avatar url --- .../extractors/MediaCCCStreamExtractor.java | 20 ++++++- .../extractors/PeertubeAccountExtractor.java | 15 +++++ .../extractors/PeertubeStreamExtractor.java | 23 ++++++++ .../extractors/SoundcloudStreamExtractor.java | 18 ++++++ .../extractors/YoutubeStreamExtractor.java | 18 ++++++ .../extractor/stream/StreamExtractor.java | 33 +++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 55 +++++++++++++++++-- 7 files changed, 176 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 894a0f0db..8eed7091f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -112,7 +112,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDashMpdUrl() { + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getDashMpdUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 81cb0afae..d5bdacc6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -75,6 +75,21 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } } + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 3cf65154d..05f1846b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -147,6 +147,29 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + value; } + @Override + public String getParentChannelUrl() throws ParsingException { + return JsonUtils.getString(json, "channel.url"); + } + + @Nonnull + @Override + public String getParentChannelName() throws ParsingException { + return JsonUtils.getString(json, "channel.displayName"); + } + + @Nonnull + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "channel.avatar.path"); + } catch (Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + @Override public String getDashMpdUrl() throws ParsingException { return ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 1c4c9cf11..d276c2b90 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -142,6 +142,24 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return SoundcloudParsingHelper.getAvatarUrl(track); } + @Nonnull + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public String getDashMpdUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 331c9000f..503d09e1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -352,6 +352,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull + @Override + public String getParentChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public String getDashMpdUrl() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 78a5fbf3d..c740ffd75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -178,6 +178,39 @@ public abstract class StreamExtractor extends Extractor { @Nonnull public abstract String getUploaderAvatarUrl() throws ParsingException; + /** + * The Url to the page of the parent chanel of the stream. This must not be a homepage, + * but the page offered by the service the extractor handles. This url will be handled by the + * ChannelExtractor, + * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects + * this url. + * + * @return the url to the page of the parent chanel of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getParentChannelUrl() throws ParsingException; + + /** + * The name of the parent chanel of the stream. + * If the name is not available you can simply return an empty string. + * + * @return the name of the parent chanel of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getParentChannelName() throws ParsingException; + + /** + * The url to the image file/profile picture/avatar of the parent chanel of the stream. + * If the url is not available you can return an empty String. + * + * @return The url of the image file of the parent chanel or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getParentChannelAvatarUrl() throws ParsingException; + /** * Get the dash mpd url. If you don't know what a dash MPD is you can read about it * here. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 0e5ff0808..a3cf3dc7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -223,6 +223,28 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + } catch (Exception e) { + streamInfo.addError(e); + } + + try { + streamInfo.setParentChannelName(extractor.getParentChannelName()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setParentChannelUrl(extractor.getParentChannelUrl()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { streamInfo.setDescription(extractor.getDescription()); } catch (Exception e) { @@ -243,11 +265,6 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } - try { - streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); - } catch (Exception e) { - streamInfo.addError(e); - } try { streamInfo.setStartPosition(extractor.getTimeStamp()); } catch (Exception e) { @@ -332,6 +349,10 @@ public class StreamInfo extends Info { private String uploaderUrl = ""; private String uploaderAvatarUrl = ""; + private String parentChannelName = ""; + private String parentChannelUrl = ""; + private String parentChannelAvatarUrl = ""; + private List videoStreams = new ArrayList<>(); private List audioStreams = new ArrayList<>(); private List videoOnlyStreams = new ArrayList<>(); @@ -486,6 +507,30 @@ public class StreamInfo extends Info { this.uploaderAvatarUrl = uploaderAvatarUrl; } + public String getParentChannelName() { + return parentChannelName; + } + + public void setParentChannelName(String parentChannelName) { + this.parentChannelName = parentChannelName; + } + + public String getParentChannelUrl() { + return parentChannelUrl; + } + + public void setParentChannelUrl(String parentChannelUrl) { + this.parentChannelUrl = parentChannelUrl; + } + + public String getParentChannelAvatarUrl() { + return parentChannelAvatarUrl; + } + + public void setParentChannelAvatarUrl(String parentChannelAvatarUrl) { + this.parentChannelAvatarUrl = parentChannelAvatarUrl; + } + public List getVideoStreams() { return videoStreams; } From a4086617719e53b293efa1e8873799ba7714cb14 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Thu, 16 Apr 2020 15:19:36 +0300 Subject: [PATCH 20/42] Fix PR review Rename "ParentChannel" to "SubChannel" Add Tests for sub-channel info fetching --- .../extractor/channel/ChannelExtractor.java | 6 +-- .../extractor/channel/ChannelInfo.java | 36 +++++++++--------- .../MediaCCCConferenceExtractor.java | 6 +-- .../extractors/MediaCCCStreamExtractor.java | 6 +-- .../extractors/PeertubeAccountExtractor.java | 6 +-- .../extractors/PeertubeChannelExtractor.java | 6 +-- .../extractors/PeertubeStreamExtractor.java | 6 +-- .../SoundcloudChannelExtractor.java | 6 +-- .../extractors/SoundcloudStreamExtractor.java | 6 +-- .../extractors/YoutubeChannelExtractor.java | 6 +-- .../extractors/YoutubeStreamExtractor.java | 6 +-- .../extractor/stream/StreamExtractor.java | 23 +++++------ .../newpipe/extractor/stream/StreamInfo.java | 36 +++++++++--------- .../PeertubeChannelExtractorTest.java | 16 ++++---- .../PeertubeStreamExtractorDefaultTest.java | 38 +++++++++++++------ 15 files changed, 113 insertions(+), 96 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 3f54f1c2c..3d3b9b1ba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -37,7 +37,7 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; - public abstract String getParentChannelName() throws ParsingException; - public abstract String getParentChannelUrl() throws ParsingException; - public abstract String getParentChannelAvatarUrl() throws ParsingException; + public abstract String getSubChannelName() throws ParsingException; + public abstract String getSubChannelUrl() throws ParsingException; + public abstract String getSubChannelAvatarUrl() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 52f18eadc..874575308 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -95,19 +95,19 @@ public class ChannelInfo extends ListInfo { } try { - info.setParentChannelName(extractor.getParentChannelName()); + info.setSubChannelName(extractor.getSubChannelName()); } catch (Exception e) { info.addError(e); } try { - info.setParentChannelUrl(extractor.getParentChannelUrl()); + info.setSubChannelUrl(extractor.getSubChannelUrl()); } catch (Exception e) { info.addError(e); } try { - info.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + info.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); } catch (Exception e) { info.addError(e); } @@ -116,37 +116,37 @@ public class ChannelInfo extends ListInfo { } private String avatarUrl; - private String parentChannelName; - private String parentChannelUrl; - private String parentChannelAvatarUrl; + private String subChannelName; + private String subChannelUrl; + private String subChannelAvatarUrl; private String bannerUrl; private String feedUrl; private long subscriberCount = -1; private String description; private String[] donationLinks; - public String getParentChannelName() { - return parentChannelName; + public String getSubChannelName() { + return subChannelName; } - public void setParentChannelName(String parentChannelName) { - this.parentChannelName = parentChannelName; + public void setSubChannelName(String subChannelName) { + this.subChannelName = subChannelName; } - public String getParentChannelUrl() { - return parentChannelUrl; + public String getSubChannelUrl() { + return subChannelUrl; } - public void setParentChannelUrl(String parentChannelUrl) { - this.parentChannelUrl = parentChannelUrl; + public void setSubChannelUrl(String subChannelUrl) { + this.subChannelUrl = subChannelUrl; } - public String getParentChannelAvatarUrl() { - return parentChannelAvatarUrl; + public String getSubChannelAvatarUrl() { + return subChannelAvatarUrl; } - public void setParentChannelAvatarUrl(String parentChannelAvatarUrl) { - this.parentChannelAvatarUrl = parentChannelAvatarUrl; + public void setSubChannelAvatarUrl(String subChannelAvatarUrl) { + this.subChannelAvatarUrl = subChannelAvatarUrl; } public String getAvatarUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index f11c13c22..4ec6bb646 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -53,17 +53,17 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 8eed7091f..93772608e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -112,19 +112,19 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index d5bdacc6a..fd82e4304 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -76,17 +76,17 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index c8f7f4cb3..ff0ff97df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -76,17 +76,17 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return JsonUtils.getString(json, "ownerAccount.name"); } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return JsonUtils.getString(json, "ownerAccount.url"); } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { String value; try { value = JsonUtils.getString(json, "ownerAccount.avatar.path"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 05f1846b2..88ef02435 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -148,19 +148,19 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return JsonUtils.getString(json, "channel.url"); } @Nonnull @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return JsonUtils.getString(json, "channel.displayName"); } @Nonnull @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { String value; try { value = JsonUtils.getString(json, "channel.avatar.path"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 1068b36f1..13afc4c7f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -84,17 +84,17 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index d276c2b90..beee2c3ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -144,19 +144,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 0e5329425..7b614bffa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -213,17 +213,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 503d09e1d..22dc78c52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -354,19 +354,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getParentChannelUrl() throws ParsingException { + public String getSubChannelUrl() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelName() throws ParsingException { + public String getSubChannelName() throws ParsingException { return ""; } @Nonnull @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index c740ffd75..2cc64ab70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -148,7 +149,7 @@ public abstract class StreamExtractor extends Extractor { /** * The Url to the page of the creator/uploader of the stream. This must not be a homepage, * but the page offered by the service the extractor handles. This url will be handled by the - * ChannelExtractor, + * {@link ChannelExtractor}, * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects * this url. * @@ -179,37 +180,37 @@ public abstract class StreamExtractor extends Extractor { public abstract String getUploaderAvatarUrl() throws ParsingException; /** - * The Url to the page of the parent chanel of the stream. This must not be a homepage, + * The Url to the page of the sub-channel of the stream. This must not be a homepage, * but the page offered by the service the extractor handles. This url will be handled by the - * ChannelExtractor, + * {@link ChannelExtractor}, * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects * this url. * - * @return the url to the page of the parent chanel of the stream or an empty String + * @return the url to the page of the sub-channel of the stream or an empty String * @throws ParsingException */ @Nonnull - public abstract String getParentChannelUrl() throws ParsingException; + public abstract String getSubChannelUrl() throws ParsingException; /** - * The name of the parent chanel of the stream. + * The name of the sub-channel of the stream. * If the name is not available you can simply return an empty string. * - * @return the name of the parent chanel of the stream or an empty String + * @return the name of the sub-channel of the stream or an empty String * @throws ParsingException */ @Nonnull - public abstract String getParentChannelName() throws ParsingException; + public abstract String getSubChannelName() throws ParsingException; /** - * The url to the image file/profile picture/avatar of the parent chanel of the stream. + * The url to the image file/profile picture/avatar of the sub-channel of the stream. * If the url is not available you can return an empty String. * - * @return The url of the image file of the parent chanel or an empty String + * @return The url of the image file of the sub-channel or an empty String * @throws ParsingException */ @Nonnull - public abstract String getParentChannelAvatarUrl() throws ParsingException; + public abstract String getSubChannelAvatarUrl() throws ParsingException; /** * Get the dash mpd url. If you don't know what a dash MPD is you can read about it diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index a3cf3dc7e..aa1001b44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -230,17 +230,17 @@ public class StreamInfo extends Info { } try { - streamInfo.setParentChannelName(extractor.getParentChannelName()); + streamInfo.setSubChannelName(extractor.getSubChannelName()); } catch (Exception e) { streamInfo.addError(e); } try { - streamInfo.setParentChannelUrl(extractor.getParentChannelUrl()); + streamInfo.setSubChannelUrl(extractor.getSubChannelUrl()); } catch (Exception e) { streamInfo.addError(e); } try { - streamInfo.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + streamInfo.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); } catch (Exception e) { streamInfo.addError(e); } @@ -349,9 +349,9 @@ public class StreamInfo extends Info { private String uploaderUrl = ""; private String uploaderAvatarUrl = ""; - private String parentChannelName = ""; - private String parentChannelUrl = ""; - private String parentChannelAvatarUrl = ""; + private String subChannelName = ""; + private String subChannelUrl = ""; + private String subChannelAvatarUrl = ""; private List videoStreams = new ArrayList<>(); private List audioStreams = new ArrayList<>(); @@ -507,28 +507,28 @@ public class StreamInfo extends Info { this.uploaderAvatarUrl = uploaderAvatarUrl; } - public String getParentChannelName() { - return parentChannelName; + public String getSubChannelName() { + return subChannelName; } - public void setParentChannelName(String parentChannelName) { - this.parentChannelName = parentChannelName; + public void setSubChannelName(String subChannelName) { + this.subChannelName = subChannelName; } - public String getParentChannelUrl() { - return parentChannelUrl; + public String getSubChannelUrl() { + return subChannelUrl; } - public void setParentChannelUrl(String parentChannelUrl) { - this.parentChannelUrl = parentChannelUrl; + public void setSubChannelUrl(String subChannelUrl) { + this.subChannelUrl = subChannelUrl; } - public String getParentChannelAvatarUrl() { - return parentChannelAvatarUrl; + public String getSubChannelAvatarUrl() { + return subChannelAvatarUrl; } - public void setParentChannelAvatarUrl(String parentChannelAvatarUrl) { - this.parentChannelAvatarUrl = parentChannelAvatarUrl; + public void setSubChannelAvatarUrl(String subChannelAvatarUrl) { + this.subChannelAvatarUrl = subChannelAvatarUrl; } public List getVideoStreams() { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index bdab43ef8..77d0623e8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -85,13 +85,13 @@ public class PeertubeChannelExtractorTest { } @Test - public void testParentChannelName() throws ParsingException { - assertEquals("libux", extractor.getParentChannelName()); + public void testSubChannelName() throws ParsingException { + assertEquals("libux", extractor.getSubChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getParentChannelUrl()); + public void testSubChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getSubChannelUrl()); } @Test @@ -192,13 +192,13 @@ public class PeertubeChannelExtractorTest { } @Test - public void testParentChannelName() throws ParsingException { - assertEquals("booteille", extractor.getParentChannelName()); + public void testSubChannelName() throws ParsingException { + assertEquals("booteille", extractor.getSubChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getParentChannelUrl()); + public void testSubChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getSubChannelUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index f5202f39d..dd4c9c449 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -87,6 +87,33 @@ public class PeertubeStreamExtractorDefaultTest { assertEquals("Framasoft", extractor.getUploaderName()); } + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetSubChannelName() throws ParsingException { + assertEquals("Les vidéos de Framasoft", extractor.getSubChannelName()); + } + + @Test + public void testGetSubChannelUrl() throws ParsingException { + assertIsSecureUrl(extractor.getSubChannelUrl()); + assertEquals("https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8", extractor.getSubChannelUrl()); + } + + @Test + public void testGetSubChannelAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getSubChannelAvatarUrl()); + } + @Test public void testGetLength() throws ParsingException { assertEquals(113, extractor.getLength()); @@ -98,22 +125,11 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } - @Test - public void testGetUploaderUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); - } - @Test public void testGetThumbnailUrl() throws ParsingException { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - @Test public void testGetVideoStreams() throws IOException, ExtractionException { assertFalse(extractor.getVideoStreams().isEmpty()); From 7320108c661ad029e7fecedabb3611143319c4c1 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 8 May 2020 16:02:59 +0200 Subject: [PATCH 21/42] Rename SubChannel in channels back to ParentChannel --- .../extractor/channel/ChannelExtractor.java | 6 ++-- .../extractor/channel/ChannelInfo.java | 36 +++++++++---------- .../MediaCCCConferenceExtractor.java | 6 ++-- .../extractors/PeertubeAccountExtractor.java | 6 ++-- .../extractors/PeertubeChannelExtractor.java | 6 ++-- .../SoundcloudChannelExtractor.java | 6 ++-- .../extractors/YoutubeChannelExtractor.java | 6 ++-- .../PeertubeChannelExtractorTest.java | 26 +++++++++----- 8 files changed, 54 insertions(+), 44 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 3d3b9b1ba..3f54f1c2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -37,7 +37,7 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; - public abstract String getSubChannelName() throws ParsingException; - public abstract String getSubChannelUrl() throws ParsingException; - public abstract String getSubChannelAvatarUrl() throws ParsingException; + public abstract String getParentChannelName() throws ParsingException; + public abstract String getParentChannelUrl() throws ParsingException; + public abstract String getParentChannelAvatarUrl() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 874575308..52f18eadc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -95,19 +95,19 @@ public class ChannelInfo extends ListInfo { } try { - info.setSubChannelName(extractor.getSubChannelName()); + info.setParentChannelName(extractor.getParentChannelName()); } catch (Exception e) { info.addError(e); } try { - info.setSubChannelUrl(extractor.getSubChannelUrl()); + info.setParentChannelUrl(extractor.getParentChannelUrl()); } catch (Exception e) { info.addError(e); } try { - info.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + info.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); } catch (Exception e) { info.addError(e); } @@ -116,37 +116,37 @@ public class ChannelInfo extends ListInfo { } private String avatarUrl; - private String subChannelName; - private String subChannelUrl; - private String subChannelAvatarUrl; + private String parentChannelName; + private String parentChannelUrl; + private String parentChannelAvatarUrl; private String bannerUrl; private String feedUrl; private long subscriberCount = -1; private String description; private String[] donationLinks; - public String getSubChannelName() { - return subChannelName; + public String getParentChannelName() { + return parentChannelName; } - public void setSubChannelName(String subChannelName) { - this.subChannelName = subChannelName; + public void setParentChannelName(String parentChannelName) { + this.parentChannelName = parentChannelName; } - public String getSubChannelUrl() { - return subChannelUrl; + public String getParentChannelUrl() { + return parentChannelUrl; } - public void setSubChannelUrl(String subChannelUrl) { - this.subChannelUrl = subChannelUrl; + public void setParentChannelUrl(String parentChannelUrl) { + this.parentChannelUrl = parentChannelUrl; } - public String getSubChannelAvatarUrl() { - return subChannelAvatarUrl; + public String getParentChannelAvatarUrl() { + return parentChannelAvatarUrl; } - public void setSubChannelAvatarUrl(String subChannelAvatarUrl) { - this.subChannelAvatarUrl = subChannelAvatarUrl; + public void setParentChannelAvatarUrl(String parentChannelAvatarUrl) { + this.parentChannelAvatarUrl = parentChannelAvatarUrl; } public String getAvatarUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 4ec6bb646..f11c13c22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -53,17 +53,17 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { } @Override - public String getSubChannelName() throws ParsingException { + public String getParentChannelName() throws ParsingException { return ""; } @Override - public String getSubChannelUrl() throws ParsingException { + public String getParentChannelUrl() throws ParsingException { return ""; } @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index fd82e4304..d5bdacc6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -76,17 +76,17 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public String getSubChannelName() throws ParsingException { + public String getParentChannelName() throws ParsingException { return ""; } @Override - public String getSubChannelUrl() throws ParsingException { + public String getParentChannelUrl() throws ParsingException { return ""; } @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index ff0ff97df..c8f7f4cb3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -76,17 +76,17 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } @Override - public String getSubChannelName() throws ParsingException { + public String getParentChannelName() throws ParsingException { return JsonUtils.getString(json, "ownerAccount.name"); } @Override - public String getSubChannelUrl() throws ParsingException { + public String getParentChannelUrl() throws ParsingException { return JsonUtils.getString(json, "ownerAccount.url"); } @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() throws ParsingException { String value; try { value = JsonUtils.getString(json, "ownerAccount.avatar.path"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 13afc4c7f..1068b36f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -84,17 +84,17 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public String getSubChannelName() throws ParsingException { + public String getParentChannelName() throws ParsingException { return ""; } @Override - public String getSubChannelUrl() throws ParsingException { + public String getParentChannelUrl() throws ParsingException { return ""; } @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 7b614bffa..0e5329425 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -213,17 +213,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public String getSubChannelName() throws ParsingException { + public String getParentChannelName() throws ParsingException { return ""; } @Override - public String getSubChannelUrl() throws ParsingException { + public String getParentChannelUrl() throws ParsingException { return ""; } @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() throws ParsingException { return ""; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 77d0623e8..010dbd7b4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -85,13 +85,18 @@ public class PeertubeChannelExtractorTest { } @Test - public void testSubChannelName() throws ParsingException { - assertEquals("libux", extractor.getSubChannelName()); + public void testParentChannelName() throws ParsingException { + assertEquals("libux", extractor.getParentChannelName()); } @Test - public void testSubChannelUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getSubChannelUrl()); + public void testParentChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getParentChannelUrl()); + } + + @Test + public void testParentChannelAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); } @Test @@ -192,13 +197,18 @@ public class PeertubeChannelExtractorTest { } @Test - public void testSubChannelName() throws ParsingException { - assertEquals("booteille", extractor.getSubChannelName()); + public void testParentChannelName() throws ParsingException { + assertEquals("booteille", extractor.getParentChannelName()); } @Test - public void testSubChannelUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getSubChannelUrl()); + public void testParentChannelUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getParentChannelUrl()); + } + + @Test + public void testParentChannelAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); } @Test From 5bab9d9fc00cc36d4efc0170efa417db91e4eb51 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 9 Apr 2020 17:51:51 +0200 Subject: [PATCH 22/42] [PeerTube] playlist support & refactoring --- .../peertube/PeertubeParsingHelper.java | 28 +++++++ .../services/peertube/PeertubeService.java | 7 +- .../extractors/PeertubeAccountExtractor.java | 37 +-------- .../extractors/PeertubeChannelExtractor.java | 38 ++------- .../extractors/PeertubeCommentsExtractor.java | 35 ++------- .../extractors/PeertubePlaylistExtractor.java | 78 +++++++++++++------ .../extractors/PeertubeSearchExtractor.java | 39 ++-------- .../extractors/PeertubeTrendingExtractor.java | 37 ++------- .../PeertubePlaylistLinkHandlerFactory.java | 13 ++-- .../PeertubeStreamLinkHandlerFactory.java | 8 +- .../PeertubePlaylistExtractorTest.java | 56 +++++++++++++ ...eertubePlaylistLinkHandlerFactoryTest.java | 11 ++- 12 files changed, 189 insertions(+), 198 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 7821aa297..50711ab97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -13,6 +14,11 @@ import java.util.TimeZone; public class PeertubeParsingHelper { + public static final String START_KEY = "start"; + public static final String COUNT_KEY = "count"; + public static final int ITEMS_PER_PAGE = 12; + public static final String START_PATTERN = "start=(\\d*)"; + private PeertubeParsingHelper() { } @@ -38,4 +44,26 @@ public class PeertubeParsingHelper { return uploadDate; } + public static String getNextPageUrl(String prevPageUrl, long total) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (Parser.RegexException e) { + return ""; + } + if (StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if (nextStart >= total) { + return ""; + } else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + nextStart); + } + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index e025c2be0..77deece79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -44,8 +44,7 @@ public class PeertubeService extends StreamingService { @Override public ListLinkHandlerFactory getPlaylistLHFactory() { - // TODO Auto-generated method stub - return null; + return PeertubePlaylistLinkHandlerFactory.getInstance(); } @Override @@ -70,7 +69,6 @@ public class PeertubeService extends StreamingService { @Override public SubscriptionExtractor getSubscriptionExtractor() { - // TODO Auto-generated method stub return null; } @@ -88,8 +86,7 @@ public class PeertubeService extends StreamingService { @Override public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { - // TODO Auto-generated method stub - return null; + return new PeertubePlaylistExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index d5bdacc6a..90783e9fb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -16,17 +16,12 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeAccountExtractor extends ChannelExtractor { +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; - private static final String START_KEY = "start"; - private static final String COUNT_KEY = "count"; - private static final int ITEMS_PER_PAGE = 12; - private static final String START_PATTERN = "start=(\\d*)"; +public class PeertubeAccountExtractor extends ChannelExtractor { private InfoItemsPage initPage; private long total; @@ -135,36 +130,12 @@ public class PeertubeAccountExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); if (json != null) { PeertubeParsingHelper.validate(json); - Number number = JsonUtils.getNumber(json, "total"); - if (number != null) this.total = number.longValue(); + total = JsonUtils.getNumber(json, "total").longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get PeerTube kiosk info"); } - return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); - } - - - private String getNextPageUrl(String prevPageUrl) { - String prevStart; - try { - prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); - } catch (RegexException e) { - return ""; - } - if (StringUtil.isBlank(prevStart)) return ""; - long nextStart = 0; - try { - nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; - } catch (NumberFormatException e) { - return ""; - } - - if (nextStart >= total) { - return ""; - } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); - } + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index c8f7f4cb3..b41e21015 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -21,12 +21,9 @@ import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeChannelExtractor extends ChannelExtractor { +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; - private static final String START_KEY = "start"; - private static final String COUNT_KEY = "count"; - private static final int ITEMS_PER_PAGE = 12; - private static final String START_PATTERN = "start=(\\d*)"; +public class PeertubeChannelExtractor extends ChannelExtractor { private InfoItemsPage initPage; private long total; @@ -141,36 +138,12 @@ public class PeertubeChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); if (json != null) { PeertubeParsingHelper.validate(json); - Number number = JsonUtils.getNumber(json, "total"); - if (number != null) this.total = number.longValue(); + this.total = JsonUtils.getNumber(json, "total").longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get PeerTube kiosk info"); } - return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); - } - - - private String getNextPageUrl(String prevPageUrl) { - String prevStart; - try { - prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); - } catch (RegexException e) { - return ""; - } - if (StringUtil.isBlank(prevStart)) return ""; - long nextStart = 0; - try { - nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; - } catch (NumberFormatException e) { - return ""; - } - - if (nextStart >= total) { - return ""; - } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); - } + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override @@ -182,8 +155,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { throw new ExtractionException("Unable to extract PeerTube channel data"); } - String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - this.initPage = getPage(pageUrl); + this.initPage = getPage(getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } private void setInitialData(String responseBody) throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index d7b16b093..61db17f59 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -13,18 +13,16 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeCommentsExtractor extends CommentsExtractor { +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; - private static final String START_KEY = "start"; - private static final String COUNT_KEY = "count"; - private static final int ITEMS_PER_PAGE = 12; - private static final String START_PATTERN = "start=(\\d*)"; +public class PeertubeCommentsExtractor extends CommentsExtractor { private InfoItemsPage initPage; private long total; @@ -83,35 +81,12 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } else { throw new ExtractionException("Unable to get peertube comments info"); } - return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - this.initPage = getPage(pageUrl); - } - - private String getNextPageUrl(String prevPageUrl) { - String prevStart; - try { - prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); - } catch (RegexException e) { - return ""; - } - if (StringUtil.isBlank(prevStart)) return ""; - long nextStart = 0; - try { - nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; - } catch (NumberFormatException e) { - return ""; - } - - if (nextStart >= total) { - return ""; - } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); - } + this.initPage = getPage(getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 1a96111ab..76509ce8c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -1,85 +1,117 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; + public class PeertubePlaylistExtractor extends PlaylistExtractor { + private JsonObject playlistInfo; + private JsonObject playlistVideos; + private String initialPageUrl; + + private long total; + public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); - // TODO Auto-generated constructor stub } @Override public String getThumbnailUrl() throws ParsingException { - // TODO Auto-generated method stub - return null; + return getBaseUrl() + playlistInfo.getString("thumbnailPath"); } @Override public String getBannerUrl() throws ParsingException { - // TODO Auto-generated method stub return null; } @Override public String getUploaderUrl() throws ParsingException { - // TODO Auto-generated method stub - return null; + return playlistInfo.getObject("ownerAccount").getString("url"); } @Override public String getUploaderName() throws ParsingException { - // TODO Auto-generated method stub - return null; + return playlistInfo.getObject("ownerAccount").getString("displayName"); } @Override public String getUploaderAvatarUrl() throws ParsingException { - // TODO Auto-generated method stub - return null; + return getBaseUrl() + playlistInfo.getObject("ownerAccount").getObject("avatar").getString("path"); } @Override public long getStreamCount() throws ParsingException { - // TODO Auto-generated method stub - return 0; + return playlistInfo.getNumber("videosLength").longValue(); } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - // TODO Auto-generated method stub - return null; + return getPage(initialPageUrl); } @Override public String getNextPageUrl() throws IOException, ExtractionException { - // TODO Auto-generated method stub - return null; + return PeertubeParsingHelper.getNextPageUrl(initialPageUrl, total); } @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - // TODO Auto-generated method stub - return null; + Response response = getDownloader().get(pageUrl); + try { + playlistVideos = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json", jpe); + } + PeertubeParsingHelper.validate(playlistVideos); + + this.total = JsonUtils.getNumber(playlistVideos, "total").longValue(); + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + JsonArray videos = playlistVideos.getArray("data"); + for (Object o : videos) { + JsonObject video = ((JsonObject) o).getObject("video"); + collector.commit(new PeertubeStreamInfoItemExtractor(video, getBaseUrl())); + } + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override - public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + try { + playlistInfo = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json", jpe); + } + PeertubeParsingHelper.validate(playlistInfo); + initialPageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; } + @Nonnull @Override public String getName() throws ParsingException { - // TODO Auto-generated method stub - return null; + return playlistInfo.getString("displayName"); } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 61fcdd6df..8313d08ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -15,18 +15,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeSearchExtractor extends SearchExtractor { +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; - private static final String START_KEY = "start"; - private static final String COUNT_KEY = "count"; - private static final int ITEMS_PER_PAGE = 12; - private static final String START_PATTERN = "start=(\\d*)"; +public class PeertubeSearchExtractor extends SearchExtractor { private InfoItemsPage initPage; private long total; @@ -88,9 +86,8 @@ public class PeertubeSearchExtractor extends SearchExtractor { } if (json != null) { - Number number = JsonUtils.getNumber(json, "total"); - if (number != null) this.total = number.longValue(); - return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); + total = JsonUtils.getNumber(json, "total").longValue(); + return new InfoItemsPage<>(collectStreamsFrom(json), PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } else { throw new ExtractionException("Unable to get peertube search info"); } @@ -98,31 +95,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - this.initPage = getPage(pageUrl); + initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } - private String getNextPageUrl(String prevPageUrl) { - String prevStart; - try { - prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); - } catch (RegexException e) { - return ""; - } - if (StringUtil.isBlank(prevStart)) return ""; - long nextStart = 0; - try { - nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; - } catch (NumberFormatException e) { - return ""; - } - - if (nextStart >= total) { - return ""; - } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); - } - } - - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index e4bc1dff8..679d2bef0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -11,20 +11,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeTrendingExtractor extends KioskExtractor { +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; - private static final String START_KEY = "start"; - private static final String COUNT_KEY = "count"; - private static final int ITEMS_PER_PAGE = 12; - private static final String START_PATTERN = "start=(\\d*)"; +public class PeertubeTrendingExtractor extends KioskExtractor { private InfoItemsPage initPage; private long total; @@ -89,35 +85,12 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } else { throw new ExtractionException("Unable to get peertube kiosk info"); } - return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - this.initPage = getPage(pageUrl); - } - - private String getNextPageUrl(String prevPageUrl) { - String prevStart; - try { - prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); - } catch (RegexException e) { - return ""; - } - if (StringUtil.isBlank(prevStart)) return ""; - long nextStart = 0; - try { - nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; - } catch (NumberFormatException e) { - return ""; - } - - if (nextStart >= total) { - return ""; - } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); - } + this.initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java index dd1bd77f5..72b1d87fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -11,8 +11,7 @@ import java.util.List; public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); - private static final String ID_PATTERN = "/video-channels/([^/?&#]*)"; - private static final String VIDEO_CHANNELS_ENDPOINT = "/api/v1/video-channels/"; + private static final String ID_PATTERN = "/videos/watch/playlist/([^/?&#]*)"; public static PeertubePlaylistLinkHandlerFactory getInstance() { return instance; @@ -26,7 +25,7 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { - return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; + return baseUrl + "/api/v1/video-playlists/" + id; } @Override @@ -34,9 +33,13 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { return Parser.matchGroup1(ID_PATTERN, url); } - @Override public boolean onAcceptUrl(final String url) { - return url.contains("/video-channels/"); + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 222dc1290..ccba0195e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -37,6 +37,12 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) throws FoundAdException { - return url.contains("/videos/"); + if (url.contains("playlist")) return false; + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java new file mode 100644 index 000000000..60db7b252 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -0,0 +1,56 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +public class PeertubePlaylistExtractorTest { + + public static class Shocking { + private static PeertubePlaylistExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (PeertubePlaylistExtractor) PeerTube + .getPlaylistExtractor("https://framatube.org/videos/watch/playlist/96b0ee2b-a5a7-4794-8769-58d8ccb79ab7"); + extractor.fetchPage(); + } + + @Test + public void testGetName() throws ParsingException { + assertEquals("Shocking !", extractor.getName()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertEquals("https://framatube.org/static/thumbnails/playlist-96b0ee2b-a5a7-4794-8769-58d8ccb79ab7.jpg", extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertEquals("https://skeptikon.fr/accounts/metadechoc", extractor.getUploaderUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertEquals("https://framatube.org/lazy-static/avatars/cd0f781d-0287-4be2-94f1-24cd732337b2.jpg", extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals("Méta de Choc", extractor.getUploaderName()); + } + + @Test + public void testGetStreamCount() throws ParsingException { + assertEquals(35, extractor.getStreamCount()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java index ac04a01ee..4b1cc38a4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -25,12 +25,17 @@ public class PeertubePlaylistLinkHandlerFactoryTest { @Test public void acceptUrlTest() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos")); + assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909")); + assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909/videos")); + assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/dacdc4ef-5160-4846-9b70-a655880da667")); + assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/96b0ee2b-a5a7-4794-8769-58d8ccb79ab7")); } @Test public void getIdFromUrl() throws ParsingException { - assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33").getId()); - assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos").getId()); + assertEquals("d8ca79f9-e4c7-4269-8183-d78ed269c909", linkHandler.getId("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909")); + assertEquals("dacdc4ef-5160-4846-9b70-a655880da667", linkHandler.getId("https://framatube.org/videos/watch/playlist/dacdc4ef-5160-4846-9b70-a655880da667")); + assertEquals("bfc145f5-1be7-48a6-9b9e-4f1967199dad", linkHandler.getId("https://framatube.org/videos/watch/playlist/bfc145f5-1be7-48a6-9b9e-4f1967199dad")); + assertEquals("96b0ee2b-a5a7-4794-8769-58d8ccb79ab7", linkHandler.getId("https://framatube.org/videos/watch/playlist/96b0ee2b-a5a7-4794-8769-58d8ccb79ab7")); } } From cf99107745a86c0f4f65a405e9eb7ebbe7673d38 Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 11 Apr 2020 15:02:53 +0200 Subject: [PATCH 23/42] address yausername change --- .../peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index ccba0195e..f181d3e71 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -37,7 +37,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) throws FoundAdException { - if (url.contains("playlist")) return false; + if (url.contains("/playlist/")) return false; try { getId(url); return true; From 1a6e92ebf6e1ba2db6076cf06660991b8972cf8f Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 9 May 2020 09:52:24 +0200 Subject: [PATCH 24/42] add getSubChannel Name, Url and AvatarUrl for playlists --- .../extractor/playlist/PlaylistExtractor.java | 7 +++++++ .../extractors/PeertubePlaylistExtractor.java | 18 ++++++++++++++++++ .../SoundcloudPlaylistExtractor.java | 18 ++++++++++++++++++ .../extractors/YoutubePlaylistExtractor.java | 18 ++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 2a901dbf0..90b89022f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -6,6 +6,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import javax.annotation.Nonnull; + public abstract class PlaylistExtractor extends ListExtractor { public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -20,4 +22,9 @@ public abstract class PlaylistExtractor extends ListExtractor { public abstract String getUploaderAvatarUrl() throws ParsingException; public abstract long getStreamCount() throws ParsingException; + + @Nonnull public abstract String getSubChannelName() throws ParsingException; + @Nonnull public abstract String getSubChannelUrl() throws ParsingException; + @Nonnull public abstract String getSubChannelAvatarUrl() throws ParsingException; + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 76509ce8c..c689096ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -63,6 +63,24 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { return playlistInfo.getNumber("videosLength").longValue(); } + @Nonnull + @Override + public String getSubChannelName() throws ParsingException { + return playlistInfo.getObject("videoChannel").getString("displayName"); + } + + @Nonnull + @Override + public String getSubChannelUrl() throws ParsingException { + return playlistInfo.getObject("videoChannel").getString("url"); + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() throws ParsingException { + return getBaseUrl() + playlistInfo.getObject("videoChannel").getObject("avatar").getString("path"); + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index a529556ee..50ad8ad69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -114,6 +114,24 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return playlist.getNumber("track_count", 0).longValue(); } + @Nonnull + @Override + public String getSubChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index de2a0e714..dcb22506d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -149,6 +149,24 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } + @Nonnull + @Override + public String getSubChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { From 7cae95bf09ddc59ec77b3a905a23278318ef15f7 Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 9 May 2020 10:15:35 +0200 Subject: [PATCH 25/42] add tests for subchannel stuff in playlists --- .../peertube/PeertubePlaylistExtractorTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java index 60db7b252..47e2960fc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -52,5 +52,21 @@ public class PeertubePlaylistExtractorTest { public void testGetStreamCount() throws ParsingException { assertEquals(35, extractor.getStreamCount()); } + + @Test + public void testGetSubChannelUrl() throws ParsingException { + assertEquals("https://skeptikon.fr/video-channels/metadechoc_channel", extractor.getSubChannelUrl()); + } + + @Test + public void testGetSubChannelName() throws ParsingException { + assertEquals("SHOCKING !", extractor.getSubChannelName()); + } + + @Test + public void testGetSubChannelAvatarUrl() throws ParsingException { + assertEquals("https://framatube.org/lazy-static/avatars/f1dcd0e8-e651-42ed-ae81-bb3bd4aff2bc.png", + extractor.getSubChannelAvatarUrl()); + } } } From 1de0876fb4ec1128abd5d05b194d3bb9757c4f1c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 9 May 2020 20:06:50 +0200 Subject: [PATCH 26/42] Fix #327 --- .../youtube/extractors/YoutubeStreamExtractor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 22dc78c52..b5cdd21db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -927,8 +927,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { streamUrl = formatData.getString("url"); } else { // this url has an encrypted signature - Map cipher = Parser.compatParseMap(formatData.getString("cipher")); - streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode); + final String cipherString = formatData.has("cipher") + ? formatData.getString("cipher") + : formatData.getString("signatureCipher"); + final Map cipher = Parser.compatParseMap(cipherString); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + + decryptSignature(cipher.get("s"), decryptionCode); } urlAndItags.put(streamUrl, itagItem); From 202a73516c55f76c6ef6967ddbfa32ee09e3a5e4 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 15 Apr 2020 18:49:58 +0200 Subject: [PATCH 27/42] refactor: add Utils.isNullOrEmpty() --- .../newpipe/extractor/ListExtractor.java | 8 +++--- .../schabi/newpipe/extractor/ListInfo.java | 4 ++- .../soundcloud/SoundcloudParsingHelper.java | 6 +++-- .../SoundcloudPlaylistExtractor.java | 4 ++- .../extractors/SoundcloudStreamExtractor.java | 3 ++- .../youtube/YoutubeParsingHelper.java | 13 +++++---- .../extractors/YoutubeChannelExtractor.java | 3 ++- .../YoutubeCommentsInfoItemExtractor.java | 2 ++ .../YoutubeMusicSearchExtractor.java | 27 ++++++++++--------- .../YoutubeStreamInfoItemExtractor.java | 5 ++-- .../extractors/YoutubeTrendingExtractor.java | 3 ++- .../newpipe/extractor/stream/StreamInfo.java | 4 ++- .../schabi/newpipe/extractor/utils/Utils.java | 12 ++++++++- .../extractor/services/DefaultTests.java | 17 +++++++----- 14 files changed, 70 insertions(+), 41 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index 0e21f04b8..aae591eb6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.Collections; @@ -9,6 +10,8 @@ import java.util.List; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + /** * Base class to extractors that have a list (e.g. playlists, users). */ @@ -63,8 +66,7 @@ public abstract class ListExtractor extends Extractor { public abstract InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException; public boolean hasNextPage() throws IOException, ExtractionException { - final String nextPageUrl = getNextPageUrl(); - return nextPageUrl != null && !nextPageUrl.isEmpty(); + return !isNullOrEmpty(getNextPageUrl()); } @Override @@ -123,7 +125,7 @@ public abstract class ListExtractor extends Extractor { } public boolean hasNextPage() { - return nextPageUrl != null && !nextPageUrl.isEmpty(); + return !isNullOrEmpty(nextPageUrl); } public List getItems() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java index 1cb42e5da..38177e1e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java @@ -4,6 +4,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public abstract class ListInfo extends Info { private List relatedItems; private String nextPageUrl = null; @@ -37,7 +39,7 @@ public abstract class ListInfo extends Info { } public boolean hasNextPage() { - return nextPageUrl != null && !nextPageUrl.isEmpty(); + return !isNullOrEmpty(nextPageUrl); } public String getNextPageUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index b24d4bf85..b8dafd834 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.*; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { @@ -42,7 +44,7 @@ public class SoundcloudParsingHelper { } public static String clientId() throws ExtractionException, IOException { - if (clientId != null && !clientId.isEmpty()) return clientId; + if (!isNullOrEmpty(clientId)) return clientId; Downloader dl = NewPipe.getDownloader(); clientId = HARDCODED_CLIENT_ID; @@ -64,7 +66,7 @@ public class SoundcloudParsingHelper { for (Element element : possibleScripts) { final String srcUrl = element.attr("src"); - if (srcUrl != null && !srcUrl.isEmpty()) { + if (!isNullOrEmpty(srcUrl)) { try { return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody()); } catch (RegexException ignored) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index a529556ee..fc3910a83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -21,6 +21,8 @@ import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private static final int streamsPerRequestedPage = 15; @@ -76,7 +78,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { for (StreamInfoItem item : infoItems.getItems()) { artworkUrl = item.getThumbnailUrl(); - if (artworkUrl != null && !artworkUrl.isEmpty()) break; + if (!isNullOrEmpty(artworkUrl)) break; } } catch (Exception ignored) { } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index beee2c3ef..431baff94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -36,6 +36,7 @@ import java.util.Locale; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -191,7 +192,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { JsonObject t = (JsonObject) transcoding; String url = t.getString("url"); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { // We can only play the mp3 format, but not handle m3u playlists / streams. // what about Opus? diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index a630ae517..02a528fbe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -29,8 +29,7 @@ import java.util.*; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.*; /* * Created by Christian Schabesberger on 02.03.16. @@ -202,7 +201,7 @@ public class YoutubeParsingHelper { * @throws ParsingException */ public static String getClientVersion() throws IOException, ExtractionException { - if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; + if (!isNullOrEmpty(clientVersion)) return clientVersion; if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; final String url = "https://www.youtube.com/results?search_query=test"; @@ -245,7 +244,7 @@ public class YoutubeParsingHelper { for (String pattern : patterns) { try { contextClientVersion = Parser.matchGroup1(pattern, html); - if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + if (!isNullOrEmpty(contextClientVersion)) { return clientVersion = contextClientVersion; } } catch (Exception ignored) { @@ -365,7 +364,7 @@ public class YoutubeParsingHelper { return "https://www.youtube.com/channel/" + browseId; } - if (canonicalBaseUrl != null && !canonicalBaseUrl.isEmpty()) { + if (!isNullOrEmpty(canonicalBaseUrl)) { return "https://www.youtube.com" + canonicalBaseUrl; } @@ -403,7 +402,7 @@ public class YoutubeParsingHelper { String text = ((JsonObject) textPart).getString("text"); if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { textBuilder.append("").append(text).append(""); continue; } @@ -497,7 +496,7 @@ public class YoutubeParsingHelper { */ public static void defaultAlertsCheck(final JsonObject initialData) throws ParsingException { final JsonArray alerts = initialData.getArray("alerts"); - if (!alerts.isEmpty()) { + if (!isNullOrEmpty(alerts)) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); final String alertText = getTextFromObject(alertRenderer.getObject("text")); final String alertType = alertRenderer.getString("type", EMPTY_STRING); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 0e5329425..d9771240f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -21,6 +21,7 @@ import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* * Created by Christian Schabesberger on 25.07.16. @@ -130,7 +131,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { if (!channelId.isEmpty()) { return channelId; - } else if (redirectedChannelId != null && !redirectedChannelId.isEmpty()) { + } else if (!isNullOrEmpty(redirectedChannelId)) { return redirectedChannelId; } else { throw new ParsingException("Could not get channel id"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index b012be7cc..9d659f5d3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -11,6 +11,8 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject json; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7e952c94b..6159ab43a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -35,6 +35,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeMusicSearchExtractor extends SearchExtractor { private JsonObject initialData; @@ -238,7 +239,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUrl() throws ParsingException { final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { return url; } throw new ParsingException("Could not get url"); @@ -248,7 +249,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get name"); @@ -258,7 +259,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public long getDuration() throws ParsingException { final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (duration != null && !duration.isEmpty()) { + if (!isNullOrEmpty(duration)) { return YoutubeParsingHelper.parseDurationString(duration); } throw new ParsingException("Could not get duration"); @@ -268,7 +269,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getUploaderName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get uploader name"); @@ -295,7 +296,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { return url; } @@ -320,7 +321,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) { + if (!isNullOrEmpty(viewCount)) { return Utils.mixedNumberWordToLong(viewCount); } throw new ParsingException("Could not get view count"); @@ -360,7 +361,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get name"); @@ -369,7 +370,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUrl() throws ParsingException { final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { return url; } throw new ParsingException("Could not get url"); @@ -379,7 +380,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public long getSubscriberCount() throws ParsingException { final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) { + if (!isNullOrEmpty(viewCount)) { return Utils.mixedNumberWordToLong(viewCount); } throw new ParsingException("Could not get subscriber count"); @@ -415,7 +416,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getName() throws ParsingException { final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get name"); @@ -424,7 +425,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUrl() throws ParsingException { final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) { + if (!isNullOrEmpty(url)) { return url; } throw new ParsingException("Could not get url"); @@ -440,7 +441,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { name = getTextFromObject(info.getArray("flexColumns").getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); } - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get uploader name"); @@ -453,7 +454,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) { + if (!isNullOrEmpty(count)) { if (count.contains("100+")) { return ITEM_COUNT_MORE_THAN_100; } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index a2d294bb7..d115fdeed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -18,6 +18,7 @@ import java.util.Date; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* * Copyright (C) Christian Schabesberger 2016 @@ -93,7 +94,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(videoInfo.getObject("title")); - if (name != null && !name.isEmpty()) return name; + if (!isNullOrEmpty(name)) return name; throw new ParsingException("Could not get name"); } @@ -186,7 +187,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } final String textualUploadDate = getTextualUploadDate(); - if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { + if (timeAgoParser != null && !isNullOrEmpty(textualUploadDate)) { try { return timeAgoParser.parse(textualUploadDate); } catch (ParsingException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 87b5561c3..ddd108e87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -39,6 +39,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; @@ -73,7 +74,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); - if (name != null && !name.isEmpty()) { + if (!isNullOrEmpty(name)) { return name; } throw new ParsingException("Could not get Trending name"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index aa1001b44..6ee60ce8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -16,6 +16,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + /* * Created by Christian Schabesberger on 26.08.15. * @@ -159,7 +161,7 @@ public class StreamInfo extends Info { streamInfo.setAudioStreams(new ArrayList()); Exception dashMpdError = null; - if (streamInfo.getDashMpdUrl() != null && !streamInfo.getDashMpdUrl().isEmpty()) { + if (!isNullOrEmpty(streamInfo.getDashMpdUrl())) { try { DashMpdParser.ParserResult result = DashMpdParser.getStreams(streamInfo); streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 76aa2944f..3029fdd25 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.util.Collection; import java.util.List; public class Utils { @@ -48,7 +49,8 @@ public class Utils { String multiplier = ""; try { multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2); - } catch(ParsingException ignored) {} + } catch (ParsingException ignored) { + } double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord) .replace(",", ".")); switch (multiplier.toUpperCase()) { @@ -186,4 +188,12 @@ public class Utils { } return uri.getProtocol() + "://" + uri.getAuthority(); } + + public static boolean isNullOrEmpty(final String str) { + return str == null || str.isEmpty(); + } + + public static boolean isNullOrEmpty(final Collection collection) { + return collection == null || collection.isEmpty(); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index a52698415..eaff6acba 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -18,6 +18,7 @@ import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.*; import static org.schabi.newpipe.extractor.StreamingService.LinkType; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class DefaultTests { public static void defaultTestListOfItems(StreamingService expectedService, List itemsList, List errors) throws ParsingException { @@ -27,8 +28,10 @@ public final class DefaultTests { for (InfoItem item : itemsList) { assertIsSecureUrl(item.getUrl()); - if (item.getThumbnailUrl() != null && !item.getThumbnailUrl().isEmpty()) { - assertIsSecureUrl(item.getThumbnailUrl()); + + final String thumbnailUrl = item.getThumbnailUrl(); + if (!isNullOrEmpty(thumbnailUrl)) { + assertIsSecureUrl(thumbnailUrl); } assertNotNull("InfoItem type not set: " + item, item.getInfoType()); assertEquals("Unexpected item service id", expectedService.getServiceId(), item.getServiceId()); @@ -39,15 +42,15 @@ public final class DefaultTests { assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); // assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); - if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) { - assertIsSecureUrl(streamInfoItem.getUploaderUrl()); - assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); + final String uploaderUrl = streamInfoItem.getUploaderUrl(); + if (!isNullOrEmpty(uploaderUrl)) { + assertIsSecureUrl(uploaderUrl); + assertExpectedLinkType(expectedService, uploaderUrl, LinkType.CHANNEL); } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); - final String textualUploadDate = streamInfoItem.getTextualUploadDate(); - if (textualUploadDate != null && !textualUploadDate.isEmpty()) { + if (!isNullOrEmpty(streamInfoItem.getTextualUploadDate())) { final DateWrapper uploadDate = streamInfoItem.getUploadDate(); assertNotNull("No parsed upload date", uploadDate); assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance())); From de26e00079adc7fcec49af6e4fc7656e1e01b91d Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 11 May 2020 11:40:24 +0200 Subject: [PATCH 28/42] changed all == null || isEmpty() to isNullOrEmpty() --- .../extractors/SoundcloudChannelExtractor.java | 3 ++- .../extractors/SoundcloudChartsExtractor.java | 3 ++- .../extractors/SoundcloudPlaylistExtractor.java | 2 +- .../services/youtube/YoutubeParsingHelper.java | 2 +- .../extractors/YoutubeChannelExtractor.java | 4 ++-- .../extractors/YoutubeCommentsExtractor.java | 3 ++- .../extractors/YoutubeMusicSearchExtractor.java | 4 ++-- .../extractors/YoutubePlaylistExtractor.java | 9 +++++---- .../extractors/YoutubeSearchExtractor.java | 5 +++-- .../extractors/YoutubeStreamExtractor.java | 17 +++++++++-------- .../YoutubeStreamInfoItemExtractor.java | 16 ++++++++-------- .../schabi/newpipe/extractor/stream/Stream.java | 4 +++- .../newpipe/extractor/stream/StreamInfo.java | 2 +- .../schabi/newpipe/extractor/utils/Utils.java | 10 +++++++++- .../services/DefaultSearchExtractorTest.java | 3 ++- .../extractor/services/DefaultTests.java | 2 +- .../search/YoutubeSearchExtractorTest.java | 3 ++- 17 files changed, 55 insertions(+), 37 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 1068b36f1..295824259 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -18,6 +18,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { @@ -132,7 +133,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index 042af5130..0c15d9c6b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -13,6 +13,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudChartsExtractor extends KioskExtractor { private StreamInfoItemsCollector collector = null; @@ -36,7 +37,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index fc3910a83..4cff6bb67 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -162,7 +162,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 02a528fbe..e78168abb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -391,7 +391,7 @@ public class YoutubeParsingHelper { * @return text in the JSON object or {@code null} */ public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { - if (textObject == null || textObject.isEmpty()) return null; + if (isNullOrEmpty(textObject)) return null; if (textObject.has("simpleText")) return textObject.getString("simpleText"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d9771240f..f5cfbc243 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -245,7 +245,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -266,7 +266,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null || continuations.isEmpty()) { + if (isNullOrEmpty(continuations)) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 5a68501ed..1dea2952b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.regex.Pattern; import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeCommentsExtractor extends CommentsExtractor { @@ -91,7 +92,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } String ajaxResponse = makeAjaxRequest(pageUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 6159ab43a..20fe8a087 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -167,7 +167,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -470,7 +470,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null || continuations.isEmpty()) { + if (isNullOrEmpty(continuations)) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index de2a0e714..b7c6d6c22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -23,6 +23,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { @@ -93,11 +94,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url == null || url.isEmpty()) { + if (isNullOrEmpty(url)) { url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") .getArray("thumbnails").getObject(0).getString("url"); - if (url == null || url.isEmpty()) throw new ParsingException("Could not get playlist thumbnail"); + if (isNullOrEmpty(url)) throw new ParsingException("Could not get playlist thumbnail"); } return fixThumbnailUrl(url); @@ -166,7 +167,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -182,7 +183,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null || continuations.isEmpty()) { + if (isNullOrEmpty(continuations)) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 732ef09ad..1ef19b820 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -19,6 +19,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* * Created by Christian Schabesberger on 22.07.2018 @@ -99,7 +100,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (isNullOrEmpty(pageUrl)) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -133,7 +134,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { - if (continuations == null || continuations.isEmpty()) { + if (isNullOrEmpty(continuations)) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b5cdd21db..b543c14f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -54,6 +54,7 @@ import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* * Created by Christian Schabesberger on 06.08.15. @@ -116,10 +117,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - if (title == null || title.isEmpty()) { + if (isNullOrEmpty(title)) { title = playerResponse.getObject("videoDetails").getString("title"); - if (title == null || title.isEmpty()) throw new ParsingException("Could not get name"); + if (isNullOrEmpty(title)) throw new ParsingException("Could not get name"); } return title; @@ -167,7 +168,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - if (textualUploadDate == null || textualUploadDate.isEmpty()) { + if (isNullOrEmpty(textualUploadDate)) { return null; } @@ -204,7 +205,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() { - if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet"); + if (isNullOrEmpty(initialData)) throw new IllegalStateException("initialData is not parsed yet"); return ageLimit; } @@ -248,10 +249,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount")); - if (views == null || views.isEmpty()) { + if (isNullOrEmpty(views)) { views = playerResponse.getObject("videoDetails").getString("viewCount"); - if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count"); + if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count"); } if (views.toLowerCase().contains("no views")) return 0; @@ -329,10 +330,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - if (uploaderName == null || uploaderName.isEmpty()) { + if (isNullOrEmpty(uploaderName)) { uploaderName = playerResponse.getObject("videoDetails").getString("author"); - if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name"); + if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name"); } return uploaderName; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index d115fdeed..0757208ee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -106,7 +106,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { String duration = getTextFromObject(videoInfo.getObject("lengthText")); - if (duration == null || duration.isEmpty()) { + if (isNullOrEmpty(duration)) { for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) { duration = getTextFromObject(((JsonObject) thumbnailOverlay) @@ -114,7 +114,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } - if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration"); + if (isNullOrEmpty(duration)) throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -124,13 +124,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { public String getUploaderName() throws ParsingException { String name = getTextFromObject(videoInfo.getObject("longBylineText")); - if (name == null || name.isEmpty()) { + if (isNullOrEmpty(name)) { name = getTextFromObject(videoInfo.getObject("ownerText")); - if (name == null || name.isEmpty()) { + if (isNullOrEmpty(name)) { name = getTextFromObject(videoInfo.getObject("shortBylineText")); - if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name"); + if (isNullOrEmpty(name)) throw new ParsingException("Could not get uploader name"); } } @@ -142,15 +142,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - if (url == null || url.isEmpty()) { + if (isNullOrEmpty(url)) { url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - if (url == null || url.isEmpty()) { + if (isNullOrEmpty(url)) { url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - if (url == null || url.isEmpty()) throw new ParsingException("Could not get uploader url"); + if (isNullOrEmpty(url)) throw new ParsingException("Could not get uploader url"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index b48eeab29..7e9908167 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.MediaFormat; import java.io.Serializable; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + /** * Creates a stream object from url, format and optional torrent url */ @@ -61,7 +63,7 @@ public abstract class Stream implements Serializable { * Check if the list already contains one stream with equals stats */ public static boolean containSimilarStream(Stream stream, List streamList) { - if (stream == null || streamList == null) return false; + if (isNullOrEmpty(streamList)) return false; for (Stream cmpStream : streamList) { if (stream.equalStats(cmpStream)) return true; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 6ee60ce8e..3878e593a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -105,7 +105,7 @@ public class StreamInfo extends Info { String name = extractor.getName(); int ageLimit = extractor.getAgeLimit(); - if ((streamType == StreamType.NONE) || (url == null || url.isEmpty()) || (id == null || id.isEmpty()) + if ((streamType == StreamType.NONE) || isNullOrEmpty(url) || (isNullOrEmpty(id)) || (name == null /* streamInfo.title can be empty of course */) || (ageLimit == -1)) { throw new ExtractionException("Some important stream information was not given."); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 3029fdd25..b1a9ab343 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.utils; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; @@ -8,6 +9,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.List; +import java.util.Map; public class Utils { @@ -72,7 +74,7 @@ public class Utils { * @param url the url to be tested */ public static void checkUrl(String pattern, String url) throws ParsingException { - if (url == null || url.isEmpty()) { + if (isNullOrEmpty(url)) { throw new IllegalArgumentException("Url can't be null or empty"); } @@ -193,7 +195,13 @@ public class Utils { return str == null || str.isEmpty(); } + // can be used for JsonArrays public static boolean isNullOrEmpty(final Collection collection) { return collection == null || collection.isEmpty(); } + + // can be used for JsonObjects + public static boolean isNullOrEmpty(final Map map) { + return map == null || map.isEmpty(); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 93c4eac70..250df78f8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -8,6 +8,7 @@ import javax.annotation.Nullable; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTest implements BaseSearchExtractorTest { @@ -25,7 +26,7 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes @Override public void testSearchSuggestion() throws Exception { final String expectedSearchSuggestion = expectedSearchSuggestion(); - if (expectedSearchSuggestion == null || expectedSearchSuggestion.isEmpty()) { + if (isNullOrEmpty(expectedSearchSuggestion)) { assertEmpty("Suggestion was expected to be empty", extractor().getSearchSuggestion()); } else { assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index eaff6acba..82f75298a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -86,7 +86,7 @@ public final class DefaultTests { public static void assertNoMoreItems(ListExtractor extractor) throws Exception { assertFalse("More items available when it shouldn't", extractor.hasNextPage()); final String nextPageUrl = extractor.getNextPageUrl(); - assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl)); } public static void assertNoDuplicatedItems(StreamingService expectedService, diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d5f2f1af2..29fab054d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -20,6 +20,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeSearchExtractorTest { public static class All extends DefaultSearchExtractorTest { @@ -170,7 +171,7 @@ public class YoutubeSearchExtractorTest { assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage()); final String nextPageUrl = nextEmptyPage.getNextPageUrl(); - assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl)); } } From adaf196c99548200beefe8ca77dda538bd8f721c Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 11 May 2020 12:54:33 +0200 Subject: [PATCH 29/42] remove unused import --- .../src/main/java/org/schabi/newpipe/extractor/utils/Utils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index b1a9ab343..fcb0d5cb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.utils; -import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; From aeb2a6ff6f12352de0695f54718d1571b8141d3f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 11 May 2020 13:05:53 +0200 Subject: [PATCH 30/42] add vscode / eclipse files to .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index bd009fd87..4f59d1718 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,9 @@ gradle-app.setting # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties + +# vscode / eclipse files +*.classpath +*.project +*.settings +**/bin \ No newline at end of file From cbf1163b800eac359fffa5b0666ddaf58a8e4be1 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 11 May 2020 21:22:36 +0200 Subject: [PATCH 31/42] ignore workspace and folder configs --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4f59d1718..db7a554c0 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ gradle-app.setting *.classpath *.project *.settings -**/bin \ No newline at end of file +**/bin +**.vscode +*.code-workspace From 102f04e29795ff3d07583373ef0398ee8365c165 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 12 May 2020 09:14:29 +0200 Subject: [PATCH 32/42] Add sub-channel stuff to PlaylistInfo --- .../extractor/playlist/PlaylistInfo.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index f8c307380..089b1de55 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -84,6 +84,21 @@ public class PlaylistInfo extends ListInfo { info.setUploaderAvatarUrl(""); uploaderParsingErrors.add(e); } + try { + info.setSubChannelUrl(extractor.getSubChannelUrl()); + } catch (Exception e) { + uploaderParsingErrors.add(e); + } + try { + info.setSubChannelName(extractor.getSubChannelName()); + } catch (Exception e) { + uploaderParsingErrors.add(e); + } + try { + info.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + } catch (Exception e) { + uploaderParsingErrors.add(e); + } try { info.setBannerUrl(extractor.getBannerUrl()); } catch (Exception e) { @@ -107,6 +122,9 @@ public class PlaylistInfo extends ListInfo { private String uploaderUrl; private String uploaderName; private String uploaderAvatarUrl; + private String subChannelUrl; + private String subChannelName; + private String subChannelAvatarUrl; private long streamCount = 0; public String getThumbnailUrl() { @@ -149,6 +167,30 @@ public class PlaylistInfo extends ListInfo { this.uploaderAvatarUrl = uploaderAvatarUrl; } + public String getSubChannelUrl() { + return subChannelUrl; + } + + public void setSubChannelUrl(String subChannelUrl) { + this.subChannelUrl = subChannelUrl; + } + + public String getSubChannelName() { + return subChannelName; + } + + public void setSubChannelName(String subChannelName) { + this.subChannelName = subChannelName; + } + + public String getSubChannelAvatarUrl() { + return subChannelAvatarUrl; + } + + public void setSubChannelAvatarUrl(String subChannelAvatarUrl) { + this.subChannelAvatarUrl = subChannelAvatarUrl; + } + public long getStreamCount() { return streamCount; } From 7ef3e80493c736a32c389c22d327e2bc71396d99 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 15 May 2020 12:04:21 +0200 Subject: [PATCH 33/42] Extract dashManifestUrl in playerResponse --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 +++- .../youtube/stream/YoutubeStreamExtractorLivestreamTest.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b543c14f3..bdc2a10f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -377,7 +377,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { String dashManifestUrl; - if (videoInfoPage.containsKey("dashmpd")) { + if (playerResponse.getObject("streamingData").isString("dashManifestUrl")) { + return playerResponse.getObject("streamingData").getString("dashManifestUrl"); + } else if (videoInfoPage.containsKey("dashmpd")) { dashManifestUrl = videoInfoPage.get("dashmpd"); } else if (playerArgs != null && playerArgs.isString("dashmpd")) { dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index f5c7fe104..0e17fe8d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -116,8 +116,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetDashMpd() throws ParsingException { - // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. - assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty()); + assertTrue(extractor.getDashMpdUrl().startsWith("https://manifest.googlevideo.com/api/manifest/dash/")); } @Test From 036cc7e26e926b41a35f71bbbebde13f1de2f13e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 2 May 2020 08:21:47 +0200 Subject: [PATCH 34/42] Upgrade jsoup --- extractor/build.gradle | 2 +- .../services/peertube/PeertubeInstance.java | 5 ++-- .../peertube/PeertubeParsingHelper.java | 7 ++--- .../extractors/PeertubeAccountExtractor.java | 5 ++-- .../extractors/PeertubeChannelExtractor.java | 5 ++-- .../extractors/PeertubeCommentsExtractor.java | 6 ++--- .../extractors/PeertubeSearchExtractor.java | 6 ++--- .../extractors/PeertubeStreamExtractor.java | 20 ++++++++++---- .../extractors/PeertubeTrendingExtractor.java | 5 ++-- .../schabi/newpipe/extractor/utils/Utils.java | 21 ++++++++++++++- .../PeertubeCommentsExtractorTest.java | 22 +++++++-------- .../youtube/YoutubeCommentsExtractorTest.java | 27 ++++++++++--------- 12 files changed, 84 insertions(+), 47 deletions(-) diff --git a/extractor/build.gradle b/extractor/build.gradle index 2138df88e..83d97df3d 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -2,7 +2,7 @@ dependencies { implementation project(':timeago-parser') implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' - implementation 'org.jsoup:jsoup:1.9.2' + implementation 'org.jsoup:jsoup:1.13.1' implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 82b675390..a29a592e0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -3,13 +3,14 @@ package org.schabi.newpipe.extractor.services.peertube; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -43,7 +44,7 @@ public class PeertubeInstance { throw new Exception("unable to configure instance " + url, e); } - if (response == null || StringUtil.isBlank(response.responseBody())) { + if (response == null || Utils.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 50711ab97..f752ecfef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -1,10 +1,11 @@ package org.schabi.newpipe.extractor.services.peertube; import com.grack.nanojson.JsonObject; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -24,7 +25,7 @@ public class PeertubeParsingHelper { public static void validate(JsonObject json) throws ContentNotAvailableException { String error = json.getString("error"); - if (!StringUtil.isBlank(error)) { + if (!Utils.isBlank(error)) { throw new ContentNotAvailableException(error); } } @@ -51,7 +52,7 @@ public class PeertubeParsingHelper { } catch (Parser.RegexException e) { return ""; } - if (StringUtil.isBlank(prevStart)) return ""; + if (Utils.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 90783e9fb..7eabcc383 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -119,7 +120,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (response != null && !StringUtil.isBlank(response.responseBody())) { + if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index b41e21015..56d1caab5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -127,7 +128,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index 61db17f59..08f682cf9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -65,7 +66,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { @@ -88,5 +89,4 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { this.initPage = getPage(getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 6ea78b378..0c8d4fef1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; @@ -84,7 +85,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { @@ -104,5 +105,4 @@ public class PeertubeSearchExtractor extends SearchExtractor { public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE); } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 88ef02435..72b5bea93 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -17,10 +17,18 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -29,6 +37,8 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import javax.annotation.Nonnull; + public class PeertubeStreamExtractor extends StreamExtractor { @@ -255,7 +265,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } @@ -292,7 +302,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !Utils.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 679d2bef0..e6f0dc69d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.jsoup.helper.StringUtil; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -69,7 +70,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fcb0d5cb4..5b70ce59c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -203,4 +203,23 @@ public class Utils { public static boolean isNullOrEmpty(final Map map) { return map == null || map.isEmpty(); } -} \ No newline at end of file + + public static boolean isWhitespace(final int c){ + return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r'; + } + + public static boolean isBlank(final String string) { + if (string == null || string.isEmpty()) { + return true; + } + + final int length = string.length(); + for (int i = 0; i < length; i++) { + if (!isWhitespace(string.codePointAt(i))) { + return false; + } + } + + return true; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index ed962b9cd..7e4923995 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import org.jsoup.helper.StringUtil; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +9,7 @@ import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.List; @@ -50,7 +50,7 @@ public class PeertubeCommentsExtractorTest { result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); String nextPage = commentsInfo.getNextPageUrl(); - while (!StringUtil.isBlank(nextPage) && !result) { + while (!Utils.isBlank(nextPage) && !result) { InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); result = findInComments(moreItems.getItems(), "Loved it!!!"); nextPage = moreItems.getNextPageUrl(); @@ -63,15 +63,15 @@ public class PeertubeCommentsExtractorTest { public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); for (CommentsInfoItem c : comments.getItems()) { - assertFalse(StringUtil.isBlank(c.getUploaderUrl())); - assertFalse(StringUtil.isBlank(c.getUploaderName())); - assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl())); - assertFalse(StringUtil.isBlank(c.getCommentId())); - assertFalse(StringUtil.isBlank(c.getCommentText())); - assertFalse(StringUtil.isBlank(c.getName())); - assertFalse(StringUtil.isBlank(c.getTextualUploadDate())); - assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); - assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(Utils.isBlank(c.getUploaderUrl())); + assertFalse(Utils.isBlank(c.getUploaderName())); + assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + assertFalse(Utils.isBlank(c.getCommentId())); + assertFalse(Utils.isBlank(c.getCommentText())); + assertFalse(Utils.isBlank(c.getName())); + assertFalse(Utils.isBlank(c.getTextualUploadDate())); + assertFalse(Utils.isBlank(c.getThumbnailUrl())); + assertFalse(Utils.isBlank(c.getUrl())); assertFalse(c.getLikeCount() != -1); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 20e49ed14..a9f6dfa3b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import org.jsoup.helper.StringUtil; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -11,11 +10,15 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeCommentsExtractorTest { @@ -71,7 +74,7 @@ public class YoutubeCommentsExtractorTest { result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); /* String nextPage = commentsInfo.getNextPageUrl(); - while (!StringUtil.isBlank(nextPage) && !result) { + while (!Utils.isBlank(nextPage) && !result) { InfoItemsPage moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); result = findInComments(moreItems.getItems(), "s1ck m3m3"); nextPage = moreItems.getNextPageUrl(); @@ -85,16 +88,16 @@ public class YoutubeCommentsExtractorTest { DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { - assertFalse(StringUtil.isBlank(c.getUploaderUrl())); - assertFalse(StringUtil.isBlank(c.getUploaderName())); - assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl())); - assertFalse(StringUtil.isBlank(c.getCommentId())); - assertFalse(StringUtil.isBlank(c.getCommentText())); - assertFalse(StringUtil.isBlank(c.getName())); - assertFalse(StringUtil.isBlank(c.getTextualUploadDate())); + assertFalse(Utils.isBlank(c.getUploaderUrl())); + assertFalse(Utils.isBlank(c.getUploaderName())); + assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + assertFalse(Utils.isBlank(c.getCommentId())); + assertFalse(Utils.isBlank(c.getCommentText())); + assertFalse(Utils.isBlank(c.getName())); + assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); - assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(Utils.isBlank(c.getThumbnailUrl())); + assertFalse(Utils.isBlank(c.getUrl())); assertFalse(c.getLikeCount() < 0); } } From aa277e1b6a841fcf10941aba6245d6372ddfa127 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 2 May 2020 08:26:48 +0200 Subject: [PATCH 35/42] Upgrade other dependencies --- extractor/build.gradle | 8 ++++---- timeago-parser/build.gradle | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/build.gradle b/extractor/build.gradle index 83d97df3d..205115870 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -3,9 +3,9 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'org.jsoup:jsoup:1.13.1' - implementation 'org.mozilla:rhino:1.7.7.1' - implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' - implementation 'org.nibor.autolink:autolink:0.8.0' + implementation 'org.mozilla:rhino:1.7.12' + implementation 'com.github.spotbugs:spotbugs-annotations:4.0.2' + implementation 'org.nibor.autolink:autolink:0.10.0' - testImplementation 'junit:junit:4.12' + testImplementation 'junit:junit:4.13' } diff --git a/timeago-parser/build.gradle b/timeago-parser/build.gradle index ff23db9ea..e324a2b4f 100644 --- a/timeago-parser/build.gradle +++ b/timeago-parser/build.gradle @@ -1,6 +1,6 @@ dependencies { - testImplementation 'junit:junit:4.12' + testImplementation 'junit:junit:4.13' implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' - implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' + implementation 'com.github.spotbugs:spotbugs-annotations:4.0.2' } From 8f71bde776336c442da66f2656e0141881fa9987 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 9 May 2020 16:47:32 +0200 Subject: [PATCH 36/42] Fix bug for YT videos longer than 1000h --- .../services/youtube/YoutubeParsingHelper.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index e78168abb..88fec7957 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -111,9 +110,7 @@ public class YoutubeParsingHelper { public static long parseDurationString(String input) throws ParsingException, NumberFormatException { - // If time separator : is not detected, try . instead - final String[] splitInput = input.contains(":") ? input.split(":") : input.split("\\."); @@ -145,10 +142,10 @@ public class YoutubeParsingHelper { default: throw new ParsingException("Error duration string with unknown format: " + input); } - return ((((Long.parseLong(days) * 24) - + Long.parseLong(hours) * 60) - + Long.parseLong(minutes)) * 60) - + Long.parseLong(seconds); + return (((Long.parseLong(Utils.removeNonDigitCharacters(days)) * 24) + + Long.parseLong(Utils.removeNonDigitCharacters(hours)) * 60) + + Long.parseLong(Utils.removeNonDigitCharacters(minutes)) * 60) + + Long.parseLong(Utils.removeNonDigitCharacters(seconds)); } public static String getFeedUrlFrom(final String channelIdOrUser) { From 384d6acd5d950b9673d65710f7290268da069912 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 9 May 2020 20:29:40 +0200 Subject: [PATCH 37/42] Add more decryption function regexes --- .../extractors/YoutubeStreamExtractor.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b543c14f3..b82735e0b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -581,14 +581,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String HTTPS = "https:"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; - private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = - "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; - private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = - "\\b([\\w$]{2})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; - private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = - "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; - private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX = - "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; + private final static String[] REGEXES = { + "\\b[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*encodeURIComponent\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", + "\\b[a-zA-Z0-9]+\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*encodeURIComponent\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", + "\\b([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", + "([a-zA-Z0-9$]+)\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", + // Obsolete patterns + "[\"']signature[\"']\\s*,\\s*([a-zA-Z0-9$]+)\\(", + "\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", + "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*(?:encodeURIComponent\\s*\\()?\\s*([a-zA-Z0-9$]+)\\(", + "\\b[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\(", + "\\b[a-zA-Z0-9]+\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\(", + "\\bc\\s*&&\\s*a\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", + "\\bc\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", + "\\bc\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(" + }; private volatile String decryptionCode = ""; @@ -765,20 +772,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { return result == null ? "" : result.toString(); } - private String getDecryptionFuncName(String playerCode) throws DecryptException { - String[] decryptionFuncNameRegexes = { - DECRYPTION_SIGNATURE_FUNCTION_REGEX_2, - DECRYPTION_SIGNATURE_FUNCTION_REGEX, - DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, - DECRYPTION_AKAMAIZED_STRING_REGEX - }; + private String getDecryptionFuncName(final String playerCode) throws DecryptException { Parser.RegexException exception = null; - for (String regex : decryptionFuncNameRegexes) { + for (final String regex : REGEXES) { try { return Parser.matchGroup1(regex, playerCode); } catch (Parser.RegexException re) { - if (exception == null) + if (exception == null) { exception = re; + } } } throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception); From 4d683e7655c2dc8dce10274db111cb0474d532ac Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 13 May 2020 17:26:07 +0200 Subject: [PATCH 38/42] Support YouTube's learning playlists --- .../extractors/YoutubePlaylistExtractor.java | 112 +++++++++++++++--- .../youtube/YoutubePlaylistExtractorTest.java | 98 +++++++++++++++ 2 files changed, 196 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index e73d69af0..e6e9fc652 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -8,16 +8,21 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; @@ -152,34 +157,47 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getSubChannelName() throws ParsingException { + public String getSubChannelName() { return ""; } @Nonnull @Override - public String getSubChannelUrl() throws ParsingException { + public String getSubChannelUrl() { return ""; } @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() { return ""; } @Nonnull @Override public InfoItemsPage getInitialPage() { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("playlistVideoListRenderer").getArray("contents"); + .getObject("itemSectionRenderer").getArray("contents"); + + if (contents.getObject(0).has("playlistSegmentRenderer")) { + for (final Object segment : contents) { + if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) { + collectTrailerFrom(collector, ((JsonObject) segment)); + } else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) { + collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer") + .getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents")); + } + } + } else if (contents.getObject(0).has("playlistVideoListRenderer")) { + final JsonArray videos = contents.getObject(0) + .getObject("playlistVideoListRenderer").getArray("contents"); + collectStreamsFrom(collector, videos); + } - collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -189,10 +207,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") + final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); @@ -200,7 +218,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFrom(JsonArray continuations) { + private String getNextPageUrlFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { return ""; } @@ -212,9 +230,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { + "&itct=" + clickTrackingParams; } - private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) { - collector.reset(); - + private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { @@ -228,4 +244,72 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } } + + private void collectTrailerFrom(final StreamInfoItemsCollector collector, + final JsonObject segment) { + collector.commit(new StreamInfoItemExtractor() { + @Override + public String getName() throws ParsingException { + return getTextFromObject(segment.getObject("playlistSegmentRenderer") + .getObject("title")); + } + + @Override + public String getUrl() throws ParsingException { + return YoutubeStreamLinkHandlerFactory.getInstance() + .fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer") + .getObject("playlistVideoPlayerRenderer").getString("videoId")) + .getUrl(); + } + + @Override + public String getThumbnailUrl() { + return null; + } + + @Override + public StreamType getStreamType() { + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } + + @Override + public long getDuration() throws ParsingException { + return YoutubeParsingHelper.parseDurationString( + getTextFromObject(segment.getObject("playlistSegmentRenderer") + .getObject("segmentAnnotation")).split("•")[0]); + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getUploaderName() throws ParsingException { + return YoutubePlaylistExtractor.this.getUploaderName(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + return YoutubePlaylistExtractor.this.getUploaderUrl(); + } + + @Nullable + @Override + public String getTextualUploadDate() { + return null; + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + }); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 65bb7f998..f2e15a3fa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -254,4 +254,102 @@ public class YoutubePlaylistExtractorTest { assertTrue("Error in the streams count", extractor.getStreamCount() > 100); } } + + public static class LearningPlaylist implements BasePlaylistExtractorTest { + private static YoutubePlaylistExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubePlaylistExtractor) YouTube + .getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + String name = extractor.getName(); + assertTrue(name, name.startsWith("Anatomy & Physiology")); + } + + @Test + public void testId() throws Exception { + assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Ignore + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // PlaylistExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertTrue(thumbnailUrl, thumbnailUrl.contains("yt")); + } + + @Ignore + @Test + public void testBannerUrl() throws Exception { + final String bannerUrl = extractor.getBannerUrl(); + assertIsSecureUrl(bannerUrl); + assertTrue(bannerUrl, bannerUrl.contains("yt")); + } + + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl()); + } + + @Test + public void testUploaderName() throws Exception { + final String uploaderName = extractor.getUploaderName(); + assertTrue(uploaderName, uploaderName.contains("CrashCourse")); + } + + @Test + public void testUploaderAvatarUrl() throws Exception { + final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); + assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt")); + } + + @Test + public void testStreamCount() throws Exception { + assertTrue("Error in the streams count", extractor.getStreamCount() > 40); + } + } } From ab77961b0a99400e86eaf422cdb098de0e22cda9 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 13 May 2020 19:34:27 +0200 Subject: [PATCH 39/42] Extract trailer thumbnail in YT learning playlists --- .../youtube/extractors/YoutubePlaylistExtractor.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index e6e9fc652..12d6d49e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -32,6 +32,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { + private JsonArray initialAjaxJson; private JsonObject initialData; private JsonObject playlistInfo; @@ -43,9 +44,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); - initialData = ajaxJson.getObject(1).getObject("response"); + initialData = initialAjaxJson.getObject(1).getObject("response"); YoutubeParsingHelper.defaultAlertsCheck(initialData); playlistInfo = getPlaylistInfo(); @@ -264,7 +265,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() { - return null; + final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse") + .getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + return fixThumbnailUrl(url); } @Override From 98055a3c3c17f2543a63d375a44c1d1f557fa76e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 9 May 2020 20:29:40 +0200 Subject: [PATCH 40/42] Revert "Add more decryption function regexes" This reverts commit 384d6acd5d950b9673d65710f7290268da069912. --- .../extractors/YoutubeStreamExtractor.java | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 292778c58..bdc2a10f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -583,21 +583,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String HTTPS = "https:"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; - private final static String[] REGEXES = { - "\\b[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*encodeURIComponent\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", - "\\b[a-zA-Z0-9]+\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*encodeURIComponent\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", - "\\b([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", - "([a-zA-Z0-9$]+)\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", - // Obsolete patterns - "[\"']signature[\"']\\s*,\\s*([a-zA-Z0-9$]+)\\(", - "\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", - "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*(?:encodeURIComponent\\s*\\()?\\s*([a-zA-Z0-9$]+)\\(", - "\\b[cs]\\s*&&\\s*[adf]\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\(", - "\\b[a-zA-Z0-9]+\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\(", - "\\bc\\s*&&\\s*a\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", - "\\bc\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(", - "\\bc\\s*&&\\s*[a-zA-Z0-9]+\\.set\\([^,]+\\s*,\\s*\\([^)]*\\)\\s*\\(\\s*([a-zA-Z0-9$]+)\\(" - }; + private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = + "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; + private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = + "\\b([\\w$]{2})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; + private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = + "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; + private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX = + "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("; private volatile String decryptionCode = ""; @@ -774,15 +767,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { return result == null ? "" : result.toString(); } - private String getDecryptionFuncName(final String playerCode) throws DecryptException { + private String getDecryptionFuncName(String playerCode) throws DecryptException { + String[] decryptionFuncNameRegexes = { + DECRYPTION_SIGNATURE_FUNCTION_REGEX_2, + DECRYPTION_SIGNATURE_FUNCTION_REGEX, + DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, + DECRYPTION_AKAMAIZED_STRING_REGEX + }; Parser.RegexException exception = null; - for (final String regex : REGEXES) { + for (String regex : decryptionFuncNameRegexes) { try { return Parser.matchGroup1(regex, playerCode); } catch (Parser.RegexException re) { - if (exception == null) { + if (exception == null) exception = re; - } } } throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception); From e39147202cdcc3caa36d090431c6b0723c0b0d9e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 30 May 2020 09:31:08 +0200 Subject: [PATCH 41/42] Fix YouTube video duration parsing --- .../extractor/services/youtube/YoutubeParsingHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 88fec7957..6df9b4b87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -142,9 +142,9 @@ public class YoutubeParsingHelper { default: throw new ParsingException("Error duration string with unknown format: " + input); } - return (((Long.parseLong(Utils.removeNonDigitCharacters(days)) * 24) + return ((((Long.parseLong(Utils.removeNonDigitCharacters(days)) * 24) + Long.parseLong(Utils.removeNonDigitCharacters(hours)) * 60) - + Long.parseLong(Utils.removeNonDigitCharacters(minutes)) * 60) + + Long.parseLong(Utils.removeNonDigitCharacters(minutes))) * 60) + Long.parseLong(Utils.removeNonDigitCharacters(seconds)); } From 3aa6b93e9128c1cdd32d308532f4cf3027bfb7dd Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 30 May 2020 10:45:39 +0200 Subject: [PATCH 42/42] Update version to 0.19.5 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 835b640a0..a45565a21 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.0'`the `dependencies` in your `build.gradle`. Replace `v0.19.0` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.5'`the `dependencies` in your `build.gradle`. Replace `v0.19.5` with the latest release. ### Testing changes diff --git a/build.gradle b/build.gradle index 611e10332..9a2f92e14 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.19.4' + version 'v0.19.5' group 'com.github.TeamNewPipe' repositories {