Use the youtubei API for YouTube mixes + update the corresponding test + do some improvements
Use the youtubei API for YouTube mixes. The corresponding has been updated because the new API breaks the tests of YoutubeMixPlaylistExtractorTest. Remove some deprecated code (the old search code with the pbj JSON) and do some other improvements.
This commit is contained in:
TiA4f8R
parent
14569c4aa9
commit
0f9e9b8b4b
|
|
@ -64,11 +64,12 @@ public class YoutubeParsingHelper {
|
|||
private YoutubeParsingHelper() {
|
||||
}
|
||||
|
||||
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
|
||||
|
||||
private static final String HARDCODED_CLIENT_VERSION = "2.20210526.07.00";
|
||||
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
|
||||
private static final String[] MOBILE_YOUTUBE_KEYS = {"AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
|
||||
"16.20.35"};
|
||||
private static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
|
||||
private static String clientVersion;
|
||||
private static String key;
|
||||
|
||||
|
|
|
|||
|
|
@ -24,15 +24,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
|
|||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
|
@ -349,9 +347,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
// Unfortunately, we have to fetch the page even if we are only getting next streams,
|
||||
// as they don't deliver enough information on their own (the channel name, for example).
|
||||
fetchPage();
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
if (!isPageFetched()) fetchPage();
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
|
||||
getExtractorLocalization());
|
||||
|
||||
|
|
@ -383,7 +385,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body);
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
|
|||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
|
|
@ -11,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
|
|||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
|
|
@ -19,19 +21,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.*;
|
||||
|
||||
/**
|
||||
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
|
||||
|
|
@ -58,13 +55,51 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
final String url = getUrl() + "&pbj=1";
|
||||
final Response response = getResponse(url, getExtractorLocalization());
|
||||
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
|
||||
initialData = ajaxJson.getObject(3).getObject("response");
|
||||
final Localization localization = getExtractorLocalization();
|
||||
final URL url = stringToURL(getUrl());
|
||||
final String mixPlaylistId = getId();
|
||||
final String videoId = getQueryValue(url, "v");
|
||||
final String playlistIndexString = getQueryValue(url, "index");
|
||||
|
||||
final byte[] body;
|
||||
if (videoId != null) {
|
||||
if (playlistIndexString != null) {
|
||||
body = JsonWriter.string(prepareJsonBuilder(localization,
|
||||
getExtractorContentCountry())
|
||||
.value("videoId", videoId)
|
||||
.value("playlistId", mixPlaylistId)
|
||||
.value("playlistIndex", Integer.parseInt(playlistIndexString))
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
} else {
|
||||
body = JsonWriter.string(prepareJsonBuilder(localization,
|
||||
getExtractorContentCountry())
|
||||
.value("videoId", videoId)
|
||||
.value("playlistId", mixPlaylistId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
}
|
||||
} else {
|
||||
body = JsonWriter.string(prepareJsonBuilder(localization,
|
||||
getExtractorContentCountry())
|
||||
.value("playlistId", mixPlaylistId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
}
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
|
||||
headers, body, localization);
|
||||
|
||||
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
||||
.getObject("playlist").getObject("playlist");
|
||||
if (isNullOrEmpty(playlistData)) throw new ExtractionException(
|
||||
"Could not get playlistData");
|
||||
cookieValue = extractCookieValue(COOKIE_NAME, response);
|
||||
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -83,10 +118,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
|
||||
} catch (final Exception e) {
|
||||
try {
|
||||
//fallback to thumbnail of current video. Always the case for channel mix
|
||||
return getThumbnailUrlFromVideoId(
|
||||
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
|
||||
.getString("videoId"));
|
||||
// Fallback to thumbnail of current video. Always the case for channel mix
|
||||
return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
|
||||
.getObject("watchEndpoint").getString("videoId"));
|
||||
} catch (final Exception ignored) {
|
||||
}
|
||||
throw new ParsingException("Could not get playlist thumbnail", e);
|
||||
|
|
@ -100,19 +134,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Override
|
||||
public String getUploaderUrl() {
|
||||
//Youtube mix are auto-generated
|
||||
// YouTube mixes are auto-generated by YouTube
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() {
|
||||
//Youtube mix are auto-generated by YouTube
|
||||
// YouTube mixes are auto-generated by YouTube
|
||||
return "YouTube";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderAvatarUrl() {
|
||||
//Youtube mix are auto-generated by YouTube
|
||||
// YouTube mixes are auto-generated by YouTube
|
||||
return "";
|
||||
}
|
||||
|
||||
|
|
@ -123,64 +157,81 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Override
|
||||
public long getStreamCount() {
|
||||
// Auto-generated playlist always start with 25 videos and are endless
|
||||
// Auto-generated playlists always start with 25 videos and are endless
|
||||
return ListExtractor.ITEM_COUNT_INFINITE;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
|
||||
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException,
|
||||
ExtractionException {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
collectStreamsFrom(collector, playlistData.getArray("contents"));
|
||||
|
||||
final Map<String, String> cookies = new HashMap<>();
|
||||
cookies.put(COOKIE_NAME, cookieValue);
|
||||
|
||||
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies));
|
||||
}
|
||||
|
||||
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
|
||||
private Page getNextPageFrom(final JsonObject playlistJson,
|
||||
final Map<String, String> cookies) throws IOException,
|
||||
ExtractionException {
|
||||
final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
|
||||
.get(playlistJson.getArray("contents").size() - 1));
|
||||
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
|
||||
throw new ExtractionException("Could not extract next page url");
|
||||
}
|
||||
|
||||
return getUrlFromNavigationEndpoint(
|
||||
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
|
||||
+ "&pbj=1";
|
||||
final JsonObject watchEndpoint = lastStream.getObject("playlistPanelVideoRenderer")
|
||||
.getObject("navigationEndpoint").getObject("watchEndpoint");
|
||||
final String playlistId = watchEndpoint.getString("playlistId");
|
||||
final String videoId = watchEndpoint.getString("videoId");
|
||||
final int index = watchEndpoint.getInt("index");
|
||||
final String params = watchEndpoint.getString("params");
|
||||
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
|
||||
getExtractorContentCountry())
|
||||
.value("videoId", videoId)
|
||||
.value("playlistId", playlistId)
|
||||
.value("playlistIndex", index)
|
||||
.value("params", params)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
|
||||
throws ExtractionException, IOException {
|
||||
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
|
||||
ExtractionException {
|
||||
if (page == null || isNullOrEmpty(page.getUrl())) {
|
||||
throw new IllegalArgumentException("Page url is empty or null");
|
||||
throw new IllegalArgumentException("Page doesn't contain an URL");
|
||||
}
|
||||
if (!page.getCookies().containsKey(COOKIE_NAME)) {
|
||||
throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing");
|
||||
throw new IllegalArgumentException("Cookie '" + COOKIE_NAME + "' is missing");
|
||||
}
|
||||
|
||||
final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());
|
||||
final JsonObject playlistJson =
|
||||
ajaxJson.getObject(3).getObject("response").getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults").getObject("playlist")
|
||||
.getObject("playlist");
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
|
||||
getExtractorLocalization());
|
||||
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||
final JsonObject playlistJson = ajaxJson.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
|
||||
final JsonArray allStreams = playlistJson.getArray("contents");
|
||||
// Sublist because youtube returns up to 24 previous streams in the mix
|
||||
// Sublist because YouTube returns up to 24 previous streams in the mix
|
||||
// +1 because the stream of "currentIndex" was already extracted in previous request
|
||||
final List<Object> newStreams =
|
||||
allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
collectStreamsFrom(collector, newStreams);
|
||||
return new InfoItemsPage<>(collector,
|
||||
new Page(getNextPageUrlFrom(playlistJson), page.getCookies()));
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(playlistJson, page.getCookies()));
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(
|
||||
@Nonnull final StreamInfoItemsCollector collector,
|
||||
@Nullable final List<Object> streams) {
|
||||
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
||||
@Nullable final List<Object> streams) {
|
||||
|
||||
if (streams == null) {
|
||||
return;
|
||||
|
|
@ -193,7 +244,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
final JsonObject streamInfo = ((JsonObject) stream)
|
||||
.getObject("playlistPanelVideoRenderer");
|
||||
if (streamInfo != null) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo,
|
||||
timeAgoParser));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -204,7 +256,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||
if (playlistId.startsWith("RDMM")) {
|
||||
videoId = playlistId.substring(4);
|
||||
} else if (playlistId.startsWith("RDCMUC")) {
|
||||
throw new ParsingException("is channel mix");
|
||||
throw new ParsingException("This playlist is a channel mix");
|
||||
} else {
|
||||
videoId = playlistId.substring(2);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,16 +21,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
|
|||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
|
|
@ -224,14 +221,15 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
|
||||
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
|
||||
getExtractorLocalization());
|
||||
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||
|
||||
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
|
||||
.getObject(0)
|
||||
.getObject("appendContinuationItemsAction")
|
||||
.getObject(0).getObject("appendContinuationItemsAction")
|
||||
.getArray("continuationItems");
|
||||
|
||||
collectStreamsFrom(collector, continuation);
|
||||
|
|
@ -259,7 +257,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body);
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -153,10 +153,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
.getObject("itemSectionRenderer");
|
||||
|
||||
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
|
||||
|
||||
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
|
||||
} else if (((JsonObject) section).has("continuationItemRenderer")) {
|
||||
nextPage = getNewNextPageFrom(((JsonObject) section)
|
||||
nextPage = getNextPageFrom(((JsonObject) section)
|
||||
.getObject("continuationItemRenderer"));
|
||||
}
|
||||
}
|
||||
|
|
@ -174,46 +172,34 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
final Localization localization = getExtractorLocalization();
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
|
||||
if (page.getId() == null) {
|
||||
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), localization);
|
||||
// @formatter:off
|
||||
final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
|
||||
getExtractorContentCountry())
|
||||
.value("continuation", page.getId())
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
|
||||
.getObject("continuationContents").getObject("itemSectionContinuation");
|
||||
final String responseBody = getValidJsonResponseBody(getDownloader().post(
|
||||
page.getUrl(), new HashMap<>(), json));
|
||||
|
||||
collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
|
||||
final JsonArray continuations = itemSectionContinuation.getArray("continuations");
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
|
||||
} else {
|
||||
// @formatter:off
|
||||
final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
|
||||
getExtractorContentCountry())
|
||||
.value("continuation", page.getId())
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final String responseBody = getValidJsonResponseBody(getDownloader().post(
|
||||
page.getUrl(), new HashMap<>(), json));
|
||||
|
||||
final JsonObject ajaxJson;
|
||||
try {
|
||||
ajaxJson = JsonParser.object().from(responseBody);
|
||||
} catch (JsonParserException e) {
|
||||
throw new ParsingException("Could not parse JSON", e);
|
||||
}
|
||||
|
||||
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
|
||||
.getObject(0).getObject("appendContinuationItemsAction")
|
||||
.getArray("continuationItems");
|
||||
|
||||
final JsonArray contents = continuationItems.getObject(0)
|
||||
.getObject("itemSectionRenderer").getArray("contents");
|
||||
collectStreamsFrom(collector, contents);
|
||||
|
||||
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1)
|
||||
.getObject("continuationItemRenderer")));
|
||||
final JsonObject ajaxJson;
|
||||
try {
|
||||
ajaxJson = JsonParser.object().from(responseBody);
|
||||
} catch (JsonParserException e) {
|
||||
throw new ParsingException("Could not parse JSON", e);
|
||||
}
|
||||
|
||||
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
|
||||
.getObject(0).getObject("appendContinuationItemsAction")
|
||||
.getArray("continuationItems");
|
||||
|
||||
final JsonArray contents = continuationItems.getObject(0)
|
||||
.getObject("itemSectionRenderer").getArray("contents");
|
||||
collectStreamsFrom(collector, contents);
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1)
|
||||
.getObject("continuationItemRenderer")));
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(final InfoItemsSearchCollector collector,
|
||||
|
|
@ -239,22 +225,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException {
|
||||
if (isNullOrEmpty(continuations)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final JsonObject nextContinuationData = continuations.getObject(0)
|
||||
.getObject("nextContinuationData");
|
||||
final String continuation = nextContinuationData.getString("continuation");
|
||||
final String clickTrackingParams = nextContinuationData
|
||||
.getString("clickTrackingParams");
|
||||
|
||||
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation="
|
||||
+ continuation + "&itct=" + clickTrackingParams);
|
||||
}
|
||||
|
||||
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
|
||||
private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
|
||||
ExtractionException {
|
||||
if (isNullOrEmpty(continuationItemRenderer)) {
|
||||
return null;
|
||||
|
|
@ -263,7 +234,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
||||
.getObject("continuationCommand").getString("token");
|
||||
|
||||
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
|
||||
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
|
||||
|
||||
return new Page(url, token);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
import org.hamcrest.MatcherAssert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Suite;
|
||||
|
|
@ -32,12 +32,13 @@ import static org.junit.Assert.assertFalse;
|
|||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
@RunWith(Suite.class)
|
||||
@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
|
||||
public class YoutubeMixPlaylistExtractorTest {
|
||||
|
||||
public static final String PBJ = "&pbj=1";
|
||||
private static final String VIDEO_ID = "_AzeUSL9lZc";
|
||||
private static final String VIDEO_TITLE =
|
||||
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
|
||||
|
|
@ -55,8 +56,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
|
||||
+ "&list=RD" + VIDEO_ID);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
|
|
@ -89,9 +90,16 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void getPage() throws Exception {
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
|
||||
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
|
||||
+ PBJ, dummyCookie));
|
||||
final byte[] body = JsonWriter.string(prepareJsonBuilder(
|
||||
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
|
||||
.value("videoId", VIDEO_ID)
|
||||
.value("playlistId", "RD" + VIDEO_ID)
|
||||
.value("params", "OAE%3D")
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
|
@ -127,7 +135,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
@Ignore
|
||||
public static class MixWithIndex {
|
||||
|
||||
private static final String INDEX = "&index=13";
|
||||
private static final int INDEX = 13;
|
||||
private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk";
|
||||
|
||||
@BeforeClass
|
||||
|
|
@ -137,9 +145,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
|
||||
+ VIDEO_ID + INDEX);
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13
|
||||
+ "&list=RD" + VIDEO_ID + "&index=" + INDEX);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
|
|
@ -167,9 +174,17 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void getPage() throws Exception {
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
|
||||
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
|
||||
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
|
||||
final byte[] body = JsonWriter.string(prepareJsonBuilder(
|
||||
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
|
||||
.value("videoId", VIDEO_ID)
|
||||
.value("playlistId", "RD" + VIDEO_ID)
|
||||
.value("playlistIndex", INDEX)
|
||||
.value("params", "OAE%3D")
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
|
@ -210,9 +225,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
|
||||
+ VIDEO_ID);
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
|
||||
+ "&list=RDMM" + VIDEO_ID);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
|
|
@ -243,9 +257,16 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void getPage() throws Exception {
|
||||
final InfoItemsPage<StreamInfoItem> streams =
|
||||
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
|
||||
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
|
||||
final byte[] body = JsonWriter.string(prepareJsonBuilder(
|
||||
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
|
||||
.value("videoId", VIDEO_ID)
|
||||
.value("playlistId", "RDMM" + VIDEO_ID)
|
||||
.value("params", "OAE%3D")
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
|
@ -291,8 +312,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
@Test(expected = IllegalArgumentException.class)
|
||||
public void getPageEmptyUrl() throws Exception {
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
|
||||
+ "&list=RD" + VIDEO_ID);
|
||||
extractor.fetchPage();
|
||||
extractor.getPage(new Page(""));
|
||||
}
|
||||
|
|
@ -300,8 +321,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
@Test(expected = ExtractionException.class)
|
||||
public void invalidVideoId() throws Exception {
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + "abcde"
|
||||
+ "&list=RD" + "abcde");
|
||||
extractor.fetchPage();
|
||||
extractor.getName();
|
||||
}
|
||||
|
|
@ -321,9 +342,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor(
|
||||
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
|
||||
+ "&list=RDCM" + CHANNEL_ID);
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
|
||||
+ "&list=RDCM" + CHANNEL_ID);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
|
|
@ -350,9 +370,16 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void getPage() throws Exception {
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
|
||||
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
|
||||
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
|
||||
final byte[] body = JsonWriter.string(prepareJsonBuilder(
|
||||
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
|
||||
.value("videoId", VIDEO_ID_OF_CHANNEL)
|
||||
.value("playlistId", "RDCM" + CHANNEL_ID)
|
||||
.value("params", "OAE%3D")
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue