be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[Youtube] Add cookies to youtube mix request 

This way youtube wont return duplicates when getting more items of the mix (but youtube can also track us)
This commit is contained in:
Xiang Rong Lin 2020-04-16 19:28:27 +02:00 committed by XiangRongLin
parent 421935401f
commit 22d2f7e400
8 changed files with 276 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -61,6 +65,12 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
/**
* The official youtube app supports intents in this format, where after the ':' is the videoId.
* Accordingly there are other apps sharing streams in this format.
*/
public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
@ -193,22 +203,22 @@ public class YoutubeParsingHelper {
}
/**
* Checks if the given playlist id is a youtube mix (auto-generated playlist)
* Ids from a youtube mix start with "RD"
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a youtube mix
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(String playlistId) {
public static boolean isYoutubeMixId(final String playlistId) {
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
}
/**
* Checks if the given playlist id is a youtube music mix (auto-generated playlist)
* Ids from a youtube music mix start with "RD"
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a youtube music mix
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(String playlistId) {
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM");
}
@ -352,7 +362,7 @@ public class YoutubeParsingHelper {
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8");
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
Map<String, List<String>> headers = new HashMap<>();
@ -436,10 +446,14 @@ public class YoutubeParsingHelper {
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId"));
}
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint")
.getInt("startTimeSeconds"));
}
return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
@ -467,7 +481,6 @@ public class YoutubeParsingHelper {
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
url = url.replaceAll("&", "&amp;");
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
continue;
}
@ -506,8 +519,8 @@ public class YoutubeParsingHelper {
public static String getValidJsonResponseBody(final Response response)
throws ParsingException, MalformedURLException {
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
throw new ContentNotAvailableException("Not found"
+ " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
}
final String responseBody = response.responseBody();
@ -527,13 +540,39 @@ public class YoutubeParsingHelper {
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null
&& responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
throw new ParsingException("Got HTML document, expected JSON response"
+ " (latest url was: \"" + response.latestUrl() + "\")");
}
return responseBody;
}
public static Response getResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
getValidJsonResponseBody(response);
return response;
}
public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("Set-Cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
@ -541,8 +580,24 @@ public class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
final String responseBody = getValidJsonResponseBody(response);
return toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
if (!isNullOrEmpty(page.getCookies())) {
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
}
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -110,7 +110,7 @@ public class YoutubeService extends StreamingService {
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {

102
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -4,8 +4,10 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -15,34 +17,50 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray;
/**
* A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of
* "youtube.com/watch?v=videoId&list=playlistId"
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
* It handles URLs in the format of
* {@code youtube.com/watch?v=videoId&list=playlistId}
*/
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
/**
* YouTube identifies mixes based on this cookie. With this information it can generate
* continuations without duplicates.
*/
private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE";
private JsonObject initialData;
private JsonObject playlistData;
private String cookieValue;
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
public YoutubeMixPlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader)
throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
final Response response = getResponse(url, getExtractorLocalization());
final JsonArray ajaxJson = toJsonArray(response.responseBody());
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
.getObject("playlist").getObject("playlist");
cookieValue = extractCookieValue(COOKIE_NAME, response);
}
@Nonnull
@ -58,16 +76,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final String playlistId = playlistData.getString("playlistId");
return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
} catch (final Exception e) {
try {
return getThumbnailUrlFromPlaylistId(playlistId);
} catch (ParsingException e) {
//fallback to thumbnail of current video. Always the case for channel mix
return getThumbnailUrlFromVideoId(
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
.getString("videoId"));
} catch (final Exception ignored) {
}
} catch (Exception e) {
throw new ParsingException("Could not get playlist thumbnail", e);
}
}
@ -104,53 +121,56 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray("contents"));
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector,
new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue)));
}
@Override
public String getNextPageUrl() throws ExtractionException {
private String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(playlistData);
}
private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
.get(playlistData.getArray("contents").size() - 1));
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
.get(playlistJson.getArray("contents").size() - 1));
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
throw new ExtractionException("Could not extract next page url");
}
//Index of video in mix is missing, but adding it doesn't appear to have any effect.
//And since the index needs to be tracked by us, it is left out
return getUrlFromNavigationEndpoint(
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+ "&pbj=1";
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+ "&pbj=1";
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws ExtractionException, IOException {
if (pageUrl == null || pageUrl.isEmpty()) {
if (page == null || page.getUrl().isEmpty()) {
throw new ExtractionException(
new IllegalArgumentException("Page url is empty or null"));
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject playlistData =
ajaxJson.getObject(3).getObject("response").getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist")
.getObject("playlist");
final JsonArray streams = playlistData.getArray("contents");
//Because continuation requests are created with the last video of previous request as start
streams.remove(0);
collectStreamsFrom(collector, streams);
return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData));
final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());
final JsonObject playlistJson =
ajaxJson.getObject(3).getObject("response").getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist")
.getObject("playlist");
final JsonArray allStreams = playlistJson.getArray("contents");
// Sublist because youtube returns up to 24 previous streams in the mix
// +1 because the stream of "currentIndex" was already extracted in previous request
final List<Object> newStreams =
allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, newStreams);
return new InfoItemsPage<>(collector,
new Page(getNextPageUrlFrom(playlistJson), page.getCookies()));
}
private void collectStreamsFrom(
@Nonnull StreamInfoItemsCollector collector,
@Nullable JsonArray streams) {
@Nonnull final StreamInfoItemsCollector collector,
@Nullable final List<Object> streams) {
if (streams == null) {
return;
@ -158,9 +178,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object stream : streams) {
for (final Object stream : streams) {
if (stream instanceof JsonObject) {
JsonObject streamInfo = ((JsonObject) stream)
final JsonObject streamInfo = ((JsonObject) stream)
.getObject("playlistPanelVideoRenderer");
if (streamInfo != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
@ -169,7 +189,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
}
}
private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException {
private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException {
final String videoId;
if (playlistId.startsWith("RDMM")) {
videoId = playlistId.substring(4);
@ -184,7 +204,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return getThumbnailUrlFromVideoId(videoId);
}
private String getThumbnailUrlFromVideoId(String videoId) {
private String getThumbnailUrlFromVideoId(final String videoId) {
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
}

52
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -14,82 +14,88 @@ import java.util.List;
public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory();
private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
new YoutubePlaylistLinkHandlerFactory();
public static YoutubePlaylistLinkHandlerFactory getInstance() {
return instance;
return INSTANCE;
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter) {
public String getUrl(final String id, final List<String> contentFilters,
final String sortFilter) {
return "https://www.youtube.com/playlist?list=" + id;
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
try {
URL urlObj = Utils.stringToURL(url);
final URL urlObj = Utils.stringToURL(url);
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj))) {
throw new ParsingException("the url given is not a Youtube-URL");
}
String path = urlObj.getPath();
final String path = urlObj.getPath();
if (!path.equals("/watch") && !path.equals("/playlist")) {
throw new ParsingException("the url given is neither a video nor a playlist URL");
}
String listID = Utils.getQueryValue(urlObj, "list");
final String listID = Utils.getQueryValue(urlObj, "list");
if (listID == null) {
throw new ParsingException("the url given does not include a playlist");
}
if (!listID.matches("[a-zA-Z0-9_-]{10,}")) {
throw new ParsingException("the list-ID given in the URL does not match the list pattern");
throw new ParsingException(
"the list-ID given in the URL does not match the list pattern");
}
// Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists
if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) {
throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported");
if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) {
throw new ContentNotSupportedException(
"YouTube Music Mix playlists are not yet supported");
}
return listID;
} catch (final Exception exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception);
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
exception);
}
}
@Override
public boolean onAcceptUrl(final String url) {
try {
String playlistId = getId(url);
//Because youtube music mix are not supported yet.
return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId);
getId(url);
} catch (ParsingException e) {
return false;
}
return true;
}
/**
* If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like
* youtube.com/watch?v=videoId&list=playlistId
* * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is
* like
* <code>https://youtube.com/watch?v=videoId&list=playlistId</code>.
* <p>Otherwise use super</p>
*/
@Override
public ListLinkHandler fromUrl(String url) throws ParsingException {
public ListLinkHandler fromUrl(final String url) throws ParsingException {
try {
URL urlObj = Utils.stringToURL(url);
String listID = Utils.getQueryValue(urlObj, "list");
final URL urlObj = Utils.stringToURL(url);
final String listID = Utils.getQueryValue(urlObj, "list");
if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) {
String videoID = Utils.getQueryValue(urlObj, "v");
if (videoID == null) {
videoID = listID.substring(2);
}
String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID;
return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url),
getSortFilter(url));
final String newUrl = "https://www.youtube.com/watch?v=" + videoID
+ "&list=" + listID;
return new ListLinkHandler(new LinkHandler(url, newUrl, listID),
getContentFilter(url),
getSortFilter(url));
}
} catch (MalformedURLException exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(),

12
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -8,6 +8,7 @@ import java.net.URL;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -260,4 +261,15 @@ public class Utils {
}
return stringBuilder.toString();
}
public static String join(final String delimiter, final String mapJoin,
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final List<String> list = new LinkedList<>();
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
.entrySet()) {
list.add(entry.getKey() + mapJoin + entry.getValue());
}
return join(delimiter, list);
}
}

176
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -2,26 +2,45 @@ package org.schabi.newpipe.extractor.services.youtube;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.junit.Assert.*;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import java.util.HashSet;
import java.util.Set;
import org.hamcrest.MatcherAssert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@RunWith(Suite.class)
@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
public class YoutubeMixPlaylistExtractorTest {
public static final String PBJ = "&pbj=1";
private static final String VIDEO_ID = "_AzeUSL9lZc";
private static final String VIDEO_TITLE =
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
private static YoutubeMixPlaylistExtractor extractor;
private static String videoId = "_AzeUSL9lZc";
private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
public static class Mix {
@ -29,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
extractor.fetchPage();
}
@ -41,81 +60,82 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getName() throws Exception {
String name = extractor.getName();
final String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(videoTitle));
assertThat(name, containsString(VIDEO_TITLE));
}
@Test
public void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, containsString("yt"));
assertThat(thumbnailUrl, containsString(videoId));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RD" + videoId));
MatcherAssert.assertThat(thumbnailUrl, containsString("yt"));
assertThat(thumbnailUrl, containsString(VIDEO_ID));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
+ PBJ));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
public void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
streams = extractor.getPage(streams.getNextPageUrl());
for (final StreamInfoItem item : streams.getItems()) {
assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
streams = extractor.getPage(streams.getNextPage());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() throws Exception {
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
public static class MixWithIndex {
private static String index = "&index=13";
private static String videoIdNumber13 = "qHtzO49SDmk";
private static final String INDEX = "&index=13";
private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId
+ index);
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
extractor.fetchPage();
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
final String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(videoTitle));
assertThat(name, containsString(VIDEO_TITLE));
}
@Test
@ -123,47 +143,47 @@ public class YoutubeMixPlaylistExtractorTest {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, containsString("yt"));
assertThat(thumbnailUrl, containsString(videoId));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RD" + videoId));
assertThat(thumbnailUrl, containsString(VIDEO_ID));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
public void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
for (final StreamInfoItem item : streams.getItems()) {
assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
streams = extractor.getPage(streams.getNextPageUrl());
streams = extractor.getPage(streams.getNextPage());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() throws Exception {
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
@ -175,7 +195,8 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId);
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
extractor.fetchPage();
}
@ -186,7 +207,7 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getName() throws Exception {
String name = extractor.getName();
final String name = extractor.getName();
assertEquals("My Mix", name);
}
@ -197,44 +218,44 @@ public class YoutubeMixPlaylistExtractorTest {
assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RDMM" + videoId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
final InfoItemsPage<StreamInfoItem> streams =
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
public void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
streams = extractor.getPage(streams.getNextPageUrl());
for (final StreamInfoItem item : streams.getItems()) {
assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
streams = extractor.getPage(streams.getNextPage());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() throws Exception {
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
@ -249,10 +270,10 @@ public class YoutubeMixPlaylistExtractorTest {
@Test(expected = ExtractionException.class)
public void getPageEmptyUrl() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
extractor.fetchPage();
extractor.getPage("");
extractor.getPage(new Page(""));
}
@Test(expected = ExtractionException.class)
@ -267,10 +288,9 @@ public class YoutubeMixPlaylistExtractorTest {
public static class ChannelMix {
private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw";
private static String videoIdOfChannel = "mnk6gnOBYIo";
private static String channelTitle = "Linus Tech Tips";
private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw";
private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo";
private static final String CHANNEL_TITLE = "Linus Tech Tips";
@BeforeClass
@ -278,15 +298,16 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId);
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
extractor.fetchPage();
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
final String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(channelTitle));
assertThat(name, containsString(CHANNEL_TITLE));
}
@Test
@ -296,30 +317,25 @@ public class YoutubeMixPlaylistExtractorTest {
assertThat(thumbnailUrl, containsString("yt"));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RDCM" + channelId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getStreamCount() throws Exception {
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java
View File

@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix
}
@Test

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java
View File

@ -62,14 +62,14 @@ public class YoutubeServiceTest {
@Test
public void getPlayListExtractorIsNormalPlaylist() throws Exception {
PlaylistExtractor extractor = service.getPlaylistExtractor(
final PlaylistExtractor extractor = service.getPlaylistExtractor(
"https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS");
assertTrue(extractor instanceof YoutubePlaylistExtractor);
}
@Test
public void getPlaylistExtractorIsMix() throws Exception {
String videoId = "_AzeUSL9lZc";
final String videoId = "_AzeUSL9lZc";
PlaylistExtractor extractor = YouTube.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
assertTrue(extractor instanceof YoutubeMixPlaylistExtractor);