diff --git a/README.md b/README.md index 92f6cd0da..27e60357f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.3'`the `dependencies` in your `build.gradle`. Replace `v0.20.3` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.6'`the `dependencies` in your `build.gradle`. Replace `v0.20.6` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. @@ -42,7 +42,7 @@ The following sites are currently supported: - YouTube - SoundCloud -- MediaCCC +- media.ccc.de - PeerTube (no P2P) - Bandcamp diff --git a/build.gradle b/build.gradle index c20b35c97..a9a217fbb 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 - version 'v0.20.3' + version 'v0.20.5' group 'com.github.TeamNewPipe' repositories { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index df1c22617..9389e255d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.util.Objects; public abstract class Extractor { /** @@ -29,12 +30,9 @@ public abstract class Extractor { private final Downloader downloader; public Extractor(final StreamingService service, final LinkHandler linkHandler) { - if (service == null) throw new NullPointerException("service is null"); - if (linkHandler == null) throw new NullPointerException("LinkHandler is null"); - this.service = service; - this.linkHandler = linkHandler; - this.downloader = NewPipe.getDownloader(); - if (downloader == null) throw new NullPointerException("downloader is null"); + this.service = Objects.requireNonNull(service, "service is null"); + this.linkHandler = Objects.requireNonNull(linkHandler, "LinkHandler is null"); + this.downloader = Objects.requireNonNull(NewPipe.getDownloader(), "downloader is null"); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index 02b890870..a3af889ac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); @@ -89,14 +90,14 @@ public class Localization implements Serializable { Localization that = (Localization) o; - if (!languageCode.equals(that.languageCode)) return false; - return countryCode != null ? countryCode.equals(that.countryCode) : that.countryCode == null; + return languageCode.equals(that.languageCode) && + Objects.equals(countryCode, that.countryCode); } @Override public int hashCode() { int result = languageCode.hashCode(); - result = 31 * result + (countryCode != null ? countryCode.hashCode() : 0); + result = 31 * result + Objects.hashCode(countryCode); return result; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 523584e60..24dd20ba3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -32,7 +32,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap public class MediaCCCService extends StreamingService { public MediaCCCService(final int id) { - super(id, "MediaCCC", asList(AUDIO, VIDEO)); + super(id, "media.ccc.de", asList(AUDIO, VIDEO)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index 9099cb1a7..cea663edb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -23,7 +25,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra @Override public long getStreamCount() { - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 8b924cf93..b36fe039a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.join; /* * Created by Christian Schabesberger on 02.03.16. @@ -110,20 +114,18 @@ public class YoutubeParsingHelper { return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") + || host.equalsIgnoreCase("vid.encryptionin.space") || host.equalsIgnoreCase("invidious.snopyta.org") - || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("yewtu.be") - || host.equalsIgnoreCase("invidious.ggc-project.de") - || host.equalsIgnoreCase("yt.maisputain.ovh") - || host.equalsIgnoreCase("invidious.13ad.de") - || host.equalsIgnoreCase("invidious.toot.koeln") - || host.equalsIgnoreCase("invidious.fdn.fr") - || host.equalsIgnoreCase("watch.nettohikari.com") - || host.equalsIgnoreCase("invidious.snwmds.net") - || host.equalsIgnoreCase("invidious.snwmds.org") - || host.equalsIgnoreCase("invidious.snwmds.com") - || host.equalsIgnoreCase("invidious.sunsetravens.com") - || host.equalsIgnoreCase("invidious.gachirangers.com"); + || host.equalsIgnoreCase("tube.connect.cafe") + || host.equalsIgnoreCase("invidious.zapashcanon.fr") + || host.equalsIgnoreCase("invidious.kavin.rocks") + || host.equalsIgnoreCase("invidious.tube") + || host.equalsIgnoreCase("invidious.site") + || host.equalsIgnoreCase("invidious.xyz") + || host.equalsIgnoreCase("vid.mint.lgbt") + || host.equalsIgnoreCase("invidiou.site") + || host.equalsIgnoreCase("invidious.fdn.fr"); } /** @@ -194,6 +196,57 @@ public class YoutubeParsingHelper { } } + /** + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a YouTube Mix + */ + public static boolean isYoutubeMixId(final String playlistId) { + return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RDAMVM" + * @param playlistId + * @return Whether given id belongs to a YouTube Music Mix + */ + public static boolean isYoutubeMusicMixId(final String playlistId) { + return playlistId.startsWith("RDAMVM"); + } + /** + * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) + * Ids from a YouTube channel Mix start with "RDCM" + * @return Whether given id belongs to a YouTube Channel Mix + */ + public static boolean isYoutubeChannelMixId(final String playlistId) { + return playlistId.startsWith("RDCM"); + } + + /** + * Extracts the video id from the playlist id for Mixes. + * @throws ParsingException If the playlistId is a Channel Mix or not a mix. + */ + public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { + if (playlistId.startsWith("RDMM")) { //My Mix + return playlistId.substring(4); + + } else if (playlistId.startsWith("RDAMVM")) { //Music mix + return playlistId.substring(6); + + } else if (playlistId.startsWith("RMCM")) { //Channel mix + //Channel mix are build with RMCM{channelId}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + + } else if (playlistId.startsWith("RD")) { // Normal mix + return playlistId.substring(2); + + } else { //not a mix + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + } + } + public static JsonObject getInitialData(String html) throws ParsingException { try { try { @@ -418,10 +471,14 @@ public class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + .getString("playlistId")); + } + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint") + .getInt("startTimeSeconds")); + } return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -487,8 +544,8 @@ public class YoutubeParsingHelper { public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); } final String responseBody = response.responseBody(); @@ -508,13 +565,39 @@ public class YoutubeParsingHelper { final String responseContentType = response.getHeader("Content-Type"); if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); } return responseBody; } + public static Response getResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(url, headers, localization); + getValidJsonResponseBody(response); + + return response; + } + + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("set-cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -522,8 +605,24 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidJsonResponseBody(response); + return toJsonArray(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + if (!isNullOrEmpty(page.getCookies())) { + headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); + } + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray toJsonArray(final String responseBody) throws ParsingException { try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 519672141..997fd0b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; @@ -109,8 +110,12 @@ public class YoutubeService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { - return new YoutubePlaylistExtractor(this, linkHandler); + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 881fbd794..fbab74d83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -86,7 +87,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index bd2af3c67..6b5a992c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -160,8 +160,15 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String findValue(String doc, String start, String end) { - final int beginIndex = doc.indexOf(start) + start.length(); - final int endIndex = doc.indexOf(end, beginIndex); - return doc.substring(beginIndex, endIndex); + final String unescaped = doc + .replaceAll("\\\\x22", "\"") + .replaceAll("\\\\x7b", "{") + .replaceAll("\\\\x7d", "}") + .replaceAll("\\\\x5b", "[") + .replaceAll("\\\\x5d", "]"); + + final int beginIndex = unescaped.indexOf(start) + start.length(); + final int endIndex = unescaped.indexOf(end, beginIndex); + return unescaped.substring(beginIndex, endIndex); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java new file mode 100644 index 000000000..d424129ef --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -0,0 +1,227 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). + * It handles URLs in the format of + * {@code youtube.com/watch?v=videoId&list=playlistId} + */ +public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + + /** + * YouTube identifies mixes based on this cookie. With this information it can generate + * continuations without duplicates. + */ + public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + + private JsonObject initialData; + private JsonObject playlistData; + private String cookieValue; + + public YoutubeMixPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; + final Response response = getResponse(url, getExtractorLocalization()); + final JsonArray ajaxJson = toJsonArray(response.responseBody()); + initialData = ajaxJson.getObject(3).getObject("response"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("playlist").getObject("playlist"); + cookieValue = extractCookieValue(COOKIE_NAME, response); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + final String name = playlistData.getString("title"); + if (name == null) { + throw new ParsingException("Could not get playlist name"); + } + return name; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + } catch (final Exception e) { + try { + //fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") + .getString("videoId")); + } catch (final Exception ignored) { + } + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated by YouTube + return "YouTube"; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated by YouTube + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + return ListExtractor.ITEM_COUNT_INFINITE; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, playlistData.getArray("contents")); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue))); + } + + private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") + .get(playlistJson.getArray("contents").size() - 1)); + if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { + throw new ExtractionException("Could not extract next page url"); + } + + return getUrlFromNavigationEndpoint( + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; + } + + @Override + public InfoItemsPage getPage(final Page page) + throws ExtractionException, IOException { + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page url is empty or null"); + } + if (!page.getCookies().containsKey(COOKIE_NAME)) { + throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing"); + } + + final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); + final JsonObject playlistJson = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray allStreams = playlistJson.getArray("contents"); + // Sublist because youtube returns up to 24 previous streams in the mix + // +1 because the stream of "currentIndex" was already extracted in previous request + final List newStreams = + allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, newStreams); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); + } + + private void collectStreamsFrom( + @Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { + + if (streams == null) { + return; + } + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (final Object stream : streams) { + if (stream instanceof JsonObject) { + final JsonObject streamInfo = ((JsonObject) stream) + .getObject("playlistPanelVideoRenderer"); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } + } + + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else if (playlistId.startsWith("RDCMUC")) { + throw new ParsingException("is channel mix"); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException("videoId is empty"); + } + return getThumbnailUrlFromVideoId(videoId); + } + + private String getThumbnailUrlFromVideoId(final String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index c55707233..1cc29eb63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -190,9 +190,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, null); } else if (contents.getObject(0).has("playlistVideoListRenderer")) { final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); - collectStreamsFrom(collector, videos.getArray("contents")); + final JsonArray videosArray = videos.getArray("contents"); + collectStreamsFrom(collector, videosArray); - nextPage = getNextPageFrom(videos.getArray("continuations")); + nextPage = getNextPageFrom(videosArray); } return new InfoItemsPage<>(collector, nextPage); @@ -207,24 +208,34 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); - final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("playlistVideoListContinuation"); + final JsonArray continuation = ajaxJson.getObject(1) + .getObject("response") + .getArray("onResponseReceivedActions") + .getObject(0) + .getObject("appendContinuationItemsAction") + .getArray("continuationItems"); - collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); + collectStreamsFrom(collector, continuation); - return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray continuations) { - if (isNullOrEmpty(continuations)) { + private Page getNextPageFrom(final JsonArray contents) { + if (isNullOrEmpty(contents)) { return null; } - final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams); + final JsonObject lastElement = contents.getObject(contents.size() - 1); + if (lastElement.has("continuationItemRenderer")) { + final String continuation = lastElement + .getObject("continuationItemRenderer") + .getObject("continuationEndpoint") + .getObject("continuationCommand") + .getString("token"); + return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation); + } else { + return null; + } } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 608ae47cd..5eaf80852 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -55,6 +55,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; @@ -861,7 +862,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } finally { Context.exit(); } - return result == null ? "" : result.toString(); + return Objects.toString(result, ""); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 56abc194b..aa2908e64 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,60 +1,72 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; -import java.net.URL; -import java.util.List; - public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); public static YoutubePlaylistLinkHandlerFactory getInstance() { - return instance; + return INSTANCE; } @Override - public String getUrl(String id, List contentFilters, String sortFilter) { + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } - String path = urlObj.getPath(); + final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } - String listID = Utils.getQueryValue(urlObj, "list"); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { throw new ParsingException("the url given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { - throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + throw new ParsingException( + "the list-ID given in the URL does not match the list pattern"); } - // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists - if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); + if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { + throw new ContentNotSupportedException( + "YouTube Music Mix playlists are not yet supported"); + } + + if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) + && Utils.getQueryValue(urlObj, "v") == null) { + //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId + throw new ContentNotSupportedException("Channel Mix without a video id are not supported"); } return listID; } catch (final Exception exception) { - throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); } } @@ -67,4 +79,33 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } return true; } + + /** + * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is + * like + * https://youtube.com/watch?v=videoId&list=playlistId. + *

Otherwise use super

+ */ + @Override + public ListLinkHandler fromUrl(final String url) throws ParsingException { + try { + final URL urlObj = Utils.stringToURL(url); + final String listID = Utils.getQueryValue(urlObj, "list"); + if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + String videoID = Utils.getQueryValue(urlObj, "v"); + if (videoID == null) { + videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); + } + final String newUrl = "https://www.youtube.com/watch?v=" + videoID + + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), + getContentFilter(url), + getSortFilter(url)); + } + } catch (MalformedURLException exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); + } + return super.fromUrl(url); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index efc06da2a..c37dff517 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -186,20 +186,18 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { case "WWW.INVIDIO.US": case "DEV.INVIDIO.US": case "INVIDIO.US": + case "VID.ENCRYPTIONIN.SPACE": case "INVIDIOUS.SNOPYTA.ORG": - case "FI.INVIDIOUS.SNOPYTA.ORG": case "YEWTU.BE": - case "INVIDIOUS.GGC-PROJECT.DE": - case "YT.MAISPUTAIN.OVH": - case "INVIDIOUS.13AD.DE": - case "INVIDIOUS.TOOT.KOELN": - case "INVIDIOUS.FDN.FR": - case "WATCH.NETTOHIKARI.COM": - case "INVIDIOUS.SNWMDS.NET": - case "INVIDIOUS.SNWMDS.ORG": - case "INVIDIOUS.SNWMDS.COM": - case "INVIDIOUS.SUNSETRAVENS.COM": - case "INVIDIOUS.GACHIRANGERS.COM": { // code-block for hooktube.com and Invidious instances + case "TUBE.CONNECT.CAFE": + case "INVIDIOUS.ZAPASHCANON.FR": + case "INVIDIOUS.KAVIN.ROCKS": + case "INVIDIOUS.TUBE": + case "INVIDIOUS.SITE": + case "INVIDIOUS.XYZ": + case "VID.MINT.LGBT": + case "INVIDIOU.SITE": + case "INVIDIOUS.FDN.FR": { // code-block for hooktube.com and Invidious instances if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 959202700..3c2bc7d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -8,6 +8,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -260,4 +261,15 @@ public class Utils { } return stringBuilder.toString(); } + + public static String join(final String delimiter, final String mapJoin, + final Map elements) { + final List list = new LinkedList<>(); + for (final Map.Entry entry : elements + .entrySet()) { + list.add(entry.getKey() + mapJoin + entry.getValue()); + } + return join(delimiter, list); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 62da50413..55669b68e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -30,7 +30,7 @@ public class YoutubeCommentsExtractorTest { private static final String url = "https://www.youtube.com/watch?v=D00Au7k3i6o"; private static YoutubeCommentsExtractor extractor; - private static final String commentContent = "sub 4 sub"; + private static final String commentContent = "Category: Education"; @BeforeClass public static void setUp() throws Exception { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java new file mode 100644 index 000000000..883218911 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -0,0 +1,345 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.hamcrest.MatcherAssert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +@RunWith(Suite.class) +@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) +public class YoutubeMixPlaylistExtractorTest { + + public static final String PBJ = "&pbj=1"; + private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_TITLE = + "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + private static final Map dummyCookie + = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); + + private static YoutubeMixPlaylistExtractor extractor; + + public static class Mix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(VIDEO_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + MatcherAssert.assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MixWithIndex { + + private static final String INDEX = "&index=13"; + private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(VIDEO_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MyMix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" + + VIDEO_ID); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertEquals("My Mix", name); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = + extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class Invalid { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = IllegalArgumentException.class) + public void getPageEmptyUrl() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + extractor.fetchPage(); + extractor.getPage(new Page("")); + } + + @Test(expected = ExtractionException.class) + public void invalidVideoId() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); + extractor.fetchPage(); + extractor.getName(); + } + } + + public static class ChannelMix { + + private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo"; + private static final String CHANNEL_TITLE = "Linus Tech Tips"; + + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(CHANNEL_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 2b03579f9..4a68f4a0a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -3,6 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; @@ -11,10 +14,17 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -23,6 +33,9 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; /** * Test for {@link YoutubePlaylistExtractor} */ +@RunWith(Suite.class) +@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class, + LearningPlaylist.class, ContinuationsTests.class}) public class YoutubePlaylistExtractorTest { public static class NotAvailable { @@ -114,7 +127,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -227,7 +240,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -324,7 +337,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -352,4 +365,34 @@ public class YoutubePlaylistExtractorTest { assertTrue("Error in the streams count", extractor.getStreamCount() > 40); } } + + public static class ContinuationsTests { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testNoContinuations() throws Exception { + final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO"); + extractor.fetchPage(); + + assertNoMoreItems(extractor); + } + + @Test + public void testOnlySingleContinuation() throws Exception { + final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX"); + extractor.fetchPage(); + + final ListExtractor.InfoItemsPage page = defaultTestMoreItems( + extractor); + assertFalse("More items available when it shouldn't", page.hasNextPage()); + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 636a646f8..e7bec115d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist - assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test @@ -105,4 +105,23 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); } + + @Test + public void fromUrlIsMixVideo() throws Exception { + final String videoId = "_AzeUSL9lZc"; + String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + + final String mixVideoId = "qHtzO49SDmk"; + url = "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + } + + @Test + public void fromUrlIsMixPlaylist() throws Exception { + final String videoId = "_AzeUSL9lZc"; + final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 6de39e5ce..4354fa258 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -26,9 +26,13 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** @@ -54,4 +58,30 @@ public class YoutubeServiceTest { public void testGetDefaultKiosk() throws Exception { assertEquals(kioskList.getDefaultKioskExtractor(null).getId(), "Trending"); } + + + @Test + public void getPlayListExtractorIsNormalPlaylist() throws Exception { + final PlaylistExtractor extractor = service.getPlaylistExtractor( + "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); + assertTrue(extractor instanceof YoutubePlaylistExtractor); + } + + @Test + public void getPlaylistExtractorIsMix() throws Exception { + final String videoId = "_AzeUSL9lZc"; + PlaylistExtractor extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + final String mixVideoId = "qHtzO49SDmk"; + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + } }