Merge pull request #1093 from AudricV/yt_support-shorts-ui-playlists
[YouTube] Support Shorts UI in playlists
This commit is contained in:
commit
7294675aea
|
@ -41,6 +41,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
// Names of some objects in JSON response frequently used in this class
|
// Names of some objects in JSON response frequently used in this class
|
||||||
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
|
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
|
||||||
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
|
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
|
||||||
|
private static final String RICH_GRID_RENDERER = "richGridRenderer";
|
||||||
|
private static final String RICH_ITEM_RENDERER = "richItemRenderer";
|
||||||
|
private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
|
||||||
private static final String SIDEBAR = "sidebar";
|
private static final String SIDEBAR = "sidebar";
|
||||||
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
|
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
|
||||||
|
|
||||||
|
@ -85,10 +88,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
* browse response (the old returns instead a sidebar one).
|
* browse response (the old returns instead a sidebar one).
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* <p>
|
|
||||||
* This new playlist UI is currently A/B tested.
|
|
||||||
* </p>
|
|
||||||
*
|
|
||||||
* @return Whether the playlist response is using only the new playlist design
|
* @return Whether the playlist response is using only the new playlist design
|
||||||
*/
|
*/
|
||||||
private boolean checkIfResponseIsNewPlaylistInterface() {
|
private boolean checkIfResponseIsNewPlaylistInterface() {
|
||||||
|
@ -327,17 +326,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
.map(content -> content.getObject("itemSectionRenderer")
|
.map(content -> content.getObject("itemSectionRenderer")
|
||||||
.getArray("contents")
|
.getArray("contents")
|
||||||
.getObject(0))
|
.getObject(0))
|
||||||
.filter(contentItemSectionRendererContents ->
|
.filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
|
||||||
contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
|
|| content.has(RICH_GRID_RENDERER))
|
||||||
|| contentItemSectionRendererContents.has(
|
|
||||||
"playlistSegmentRenderer"))
|
|
||||||
.findFirst()
|
.findFirst()
|
||||||
.orElse(null);
|
.orElse(null);
|
||||||
|
|
||||||
if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
|
if (videoPlaylistObject != null) {
|
||||||
final JsonArray videosArray = videoPlaylistObject
|
final JsonObject renderer;
|
||||||
.getObject(PLAYLIST_VIDEO_LIST_RENDERER)
|
if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
|
||||||
.getArray("contents");
|
renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
|
||||||
|
} else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
|
||||||
|
renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
|
||||||
|
} else {
|
||||||
|
return new InfoItemsPage<>(collector, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
final JsonArray videosArray = renderer.getArray("contents");
|
||||||
collectStreamsFrom(collector, videosArray);
|
collectStreamsFrom(collector, videosArray);
|
||||||
|
|
||||||
nextPage = getNextPageFrom(videosArray);
|
nextPage = getNextPageFrom(videosArray);
|
||||||
|
@ -399,14 +403,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
||||||
@Nonnull final JsonArray videos) {
|
@Nonnull final JsonArray videos) {
|
||||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||||
|
|
||||||
videos.stream()
|
videos.stream()
|
||||||
.filter(JsonObject.class::isInstance)
|
.filter(JsonObject.class::isInstance)
|
||||||
.map(JsonObject.class::cast)
|
.map(JsonObject.class::cast)
|
||||||
.filter(video -> video.has(PLAYLIST_VIDEO_RENDERER))
|
.forEach(video -> {
|
||||||
.map(video -> new YoutubeStreamInfoItemExtractor(
|
if (video.has(PLAYLIST_VIDEO_RENDERER)) {
|
||||||
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser))
|
collector.commit(new YoutubeStreamInfoItemExtractor(
|
||||||
.forEachOrdered(collector::commit);
|
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser));
|
||||||
|
} else if (video.has(RICH_ITEM_RENDERER)) {
|
||||||
|
final JsonObject richItemRenderer = video.getObject(RICH_ITEM_RENDERER);
|
||||||
|
if (richItemRenderer.has("content")) {
|
||||||
|
final JsonObject richItemRendererContent =
|
||||||
|
richItemRenderer.getObject("content");
|
||||||
|
if (richItemRendererContent.has(REEL_ITEM_RENDERER)) {
|
||||||
|
collector.commit(new YoutubeReelInfoItemExtractor(
|
||||||
|
richItemRendererContent.getObject(REEL_ITEM_RENDERER),
|
||||||
|
timeAgoParser));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nonnull
|
@Nonnull
|
||||||
|
|
|
@ -56,7 +56,7 @@ public class ExtractorAsserts {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
|
public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
|
||||||
assertNotNull(message, stringToCheck);
|
assertNotNull(stringToCheck, message);
|
||||||
assertFalse(stringToCheck.isEmpty(), message);
|
assertFalse(stringToCheck.isEmpty(), message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -38,9 +38,6 @@ public final class DefaultTests {
|
||||||
|
|
||||||
if (item instanceof StreamInfoItem) {
|
if (item instanceof StreamInfoItem) {
|
||||||
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
|
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
|
||||||
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
|
|
||||||
|
|
||||||
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
|
|
||||||
final String uploaderUrl = streamInfoItem.getUploaderUrl();
|
final String uploaderUrl = streamInfoItem.getUploaderUrl();
|
||||||
if (!isNullOrEmpty(uploaderUrl)) {
|
if (!isNullOrEmpty(uploaderUrl)) {
|
||||||
assertIsSecureUrl(uploaderUrl);
|
assertIsSecureUrl(uploaderUrl);
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
|
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
|
||||||
import org.schabi.newpipe.extractor.stream.Description;
|
import org.schabi.newpipe.extractor.stream.Description;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -416,6 +417,120 @@ public class YoutubePlaylistExtractorTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static class ShortsUI implements BasePlaylistExtractorTest {
|
||||||
|
|
||||||
|
private static PlaylistExtractor extractor;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void setUp() throws Exception {
|
||||||
|
YoutubeTestsUtils.ensureStateless();
|
||||||
|
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "shortsUI"));
|
||||||
|
extractor = YouTube.getPlaylistExtractor(
|
||||||
|
"https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ");
|
||||||
|
extractor.fetchPage();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testServiceId() throws Exception {
|
||||||
|
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testName() throws Exception {
|
||||||
|
assertEquals("Short videos", extractor.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testId() throws Exception {
|
||||||
|
assertEquals("UUSHBR8-60-B28hp2BmDPdntcQ", extractor.getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testUrl() throws Exception {
|
||||||
|
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
|
||||||
|
extractor.getUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testOriginalUrl() throws Exception {
|
||||||
|
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
|
||||||
|
extractor.getOriginalUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testRelatedItems() throws Exception {
|
||||||
|
defaultTestRelatedItems(extractor);
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: enable test when continuations are available
|
||||||
|
@Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
|
||||||
|
+ "items: this is a bug on YouTube's side, which is not related to the requirement "
|
||||||
|
+ "of a valid visitorData like it is for Shorts channel tab")
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testMoreRelatedItems() throws Exception {
|
||||||
|
defaultTestMoreItems(extractor);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testThumbnailUrl() throws Exception {
|
||||||
|
final String thumbnailUrl = extractor.getThumbnailUrl();
|
||||||
|
assertIsSecureUrl(thumbnailUrl);
|
||||||
|
ExtractorAsserts.assertContains("yt", thumbnailUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testBannerUrl() throws Exception {
|
||||||
|
final String thumbnailUrl = extractor.getThumbnailUrl();
|
||||||
|
assertIsSecureUrl(thumbnailUrl);
|
||||||
|
ExtractorAsserts.assertContains("yt", thumbnailUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testUploaderName() throws Exception {
|
||||||
|
assertEquals("YouTube", extractor.getUploaderName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testUploaderAvatarUrl() throws Exception {
|
||||||
|
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
|
||||||
|
ExtractorAsserts.assertContains("yt", uploaderAvatarUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testStreamCount() throws Exception {
|
||||||
|
ExtractorAsserts.assertGreater(250, extractor.getStreamCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Override
|
||||||
|
public void testUploaderVerified() throws Exception {
|
||||||
|
// YouTube doesn't provide this information for playlists
|
||||||
|
assertFalse(extractor.isUploaderVerified());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void getPlaylistType() throws ParsingException {
|
||||||
|
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testDescription() throws ParsingException {
|
||||||
|
assertTrue(Utils.isNullOrEmpty(extractor.getDescription().getContent()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static class ContinuationsTests {
|
public static class ContinuationsTests {
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
|
|
|
@ -0,0 +1,85 @@
|
||||||
|
{
|
||||||
|
"request": {
|
||||||
|
"httpMethod": "GET",
|
||||||
|
"url": "https://www.youtube.com/sw.js",
|
||||||
|
"headers": {
|
||||||
|
"Referer": [
|
||||||
|
"https://www.youtube.com"
|
||||||
|
],
|
||||||
|
"Origin": [
|
||||||
|
"https://www.youtube.com"
|
||||||
|
],
|
||||||
|
"Accept-Language": [
|
||||||
|
"en-GB, en;q\u003d0.9"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"localization": {
|
||||||
|
"languageCode": "en",
|
||||||
|
"countryCode": "GB"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"response": {
|
||||||
|
"responseCode": 200,
|
||||||
|
"responseMessage": "",
|
||||||
|
"responseHeaders": {
|
||||||
|
"access-control-allow-credentials": [
|
||||||
|
"true"
|
||||||
|
],
|
||||||
|
"access-control-allow-origin": [
|
||||||
|
"https://www.youtube.com"
|
||||||
|
],
|
||||||
|
"alt-svc": [
|
||||||
|
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
|
||||||
|
],
|
||||||
|
"cache-control": [
|
||||||
|
"private, max-age\u003d0"
|
||||||
|
],
|
||||||
|
"content-type": [
|
||||||
|
"text/javascript; charset\u003dutf-8"
|
||||||
|
],
|
||||||
|
"cross-origin-opener-policy-report-only": [
|
||||||
|
"same-origin; report-to\u003d\"youtube_main\""
|
||||||
|
],
|
||||||
|
"date": [
|
||||||
|
"Mon, 07 Aug 2023 17:06:40 GMT"
|
||||||
|
],
|
||||||
|
"expires": [
|
||||||
|
"Mon, 07 Aug 2023 17:06:40 GMT"
|
||||||
|
],
|
||||||
|
"origin-trial": [
|
||||||
|
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
|
||||||
|
],
|
||||||
|
"p3p": [
|
||||||
|
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
|
||||||
|
],
|
||||||
|
"permissions-policy": [
|
||||||
|
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
|
||||||
|
],
|
||||||
|
"report-to": [
|
||||||
|
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
|
||||||
|
],
|
||||||
|
"server": [
|
||||||
|
"ESF"
|
||||||
|
],
|
||||||
|
"set-cookie": [
|
||||||
|
"YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||||
|
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||||
|
"CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
|
||||||
|
],
|
||||||
|
"strict-transport-security": [
|
||||||
|
"max-age\u003d31536000"
|
||||||
|
],
|
||||||
|
"x-content-type-options": [
|
||||||
|
"nosniff"
|
||||||
|
],
|
||||||
|
"x-frame-options": [
|
||||||
|
"SAMEORIGIN"
|
||||||
|
],
|
||||||
|
"x-xss-protection": [
|
||||||
|
"0"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
|
||||||
|
"latestUrl": "https://www.youtube.com/sw.js"
|
||||||
|
}
|
||||||
|
}
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue