be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1093 from AudricV/yt_support-shorts-ui-playlists 

[YouTube] Support Shorts UI in playlists
This commit is contained in:
Stypox 2023-08-12 11:11:36 +02:00 committed by GitHub
parent 93a90b816d 698c710685
commit 7294675aea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 700 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -41,6 +41,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
// Names of some objects in JSON response frequently used in this class
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
private static final String RICH_GRID_RENDERER = "richGridRenderer";
private static final String RICH_ITEM_RENDERER = "richItemRenderer";
private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
private static final String SIDEBAR = "sidebar";
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
@ -85,10 +88,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
* browse response (the old returns instead a sidebar one).
* </p>
*
* <p>
* This new playlist UI is currently A/B tested.
* </p>
*
* @return Whether the playlist response is using only the new playlist design
*/
private boolean checkIfResponseIsNewPlaylistInterface() {
@ -327,17 +326,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.map(content -> content.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0))
.filter(contentItemSectionRendererContents ->
contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
|| contentItemSectionRendererContents.has(
"playlistSegmentRenderer"))
.filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
|| content.has(RICH_GRID_RENDERER))
.findFirst()
.orElse(null);
if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
final JsonArray videosArray = videoPlaylistObject
.getObject(PLAYLIST_VIDEO_LIST_RENDERER)
.getArray("contents");
if (videoPlaylistObject != null) {
final JsonObject renderer;
if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
} else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
} else {
return new InfoItemsPage<>(collector, null);
}
final JsonArray videosArray = renderer.getArray("contents");
collectStreamsFrom(collector, videosArray);
nextPage = getNextPageFrom(videosArray);
@ -399,14 +403,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
videos.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(video -> video.has(PLAYLIST_VIDEO_RENDERER))
.map(video -> new YoutubeStreamInfoItemExtractor(
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser))
.forEachOrdered(collector::commit);
.forEach(video -> {
if (video.has(PLAYLIST_VIDEO_RENDERER)) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser));
} else if (video.has(RICH_ITEM_RENDERER)) {
final JsonObject richItemRenderer = video.getObject(RICH_ITEM_RENDERER);
if (richItemRenderer.has("content")) {
final JsonObject richItemRendererContent =
richItemRenderer.getObject("content");
if (richItemRendererContent.has(REEL_ITEM_RENDERER)) {
collector.commit(new YoutubeReelInfoItemExtractor(
richItemRendererContent.getObject(REEL_ITEM_RENDERER),
timeAgoParser));
}
}
}
});
}
@Nonnull

2
extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
View File

@ -56,7 +56,7 @@ public class ExtractorAsserts {
}
public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
assertNotNull(message, stringToCheck);
assertNotNull(stringToCheck, message);
assertFalse(stringToCheck.isEmpty(), message);
}

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@ -38,9 +38,6 @@ public final class DefaultTests {
if (item instanceof StreamInfoItem) {
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
final String uploaderUrl = streamInfoItem.getUploaderUrl();
if (!isNullOrEmpty(uploaderUrl)) {
assertIsSecureUrl(uploaderUrl);

115
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -28,6 +28,7 @@ import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -416,6 +417,120 @@ public class YoutubePlaylistExtractorTest {
}
}
static class ShortsUI implements BasePlaylistExtractorTest {
private static PlaylistExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "shortsUI"));
extractor = YouTube.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ");
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() throws Exception {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals("Short videos", extractor.getName());
}
@Test
@Override
public void testId() throws Exception {
assertEquals("UUSHBR8-60-B28hp2BmDPdntcQ", extractor.getId());
}
@Test
@Override
public void testUrl() throws Exception {
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
// TODO: enable test when continuations are available
@Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
+ "items: this is a bug on YouTube's side, which is not related to the requirement "
+ "of a valid visitorData like it is for Shorts channel tab")
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
@Test
@Override
public void testThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
ExtractorAsserts.assertContains("yt", thumbnailUrl);
}
@Test
@Override
public void testBannerUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
ExtractorAsserts.assertContains("yt", thumbnailUrl);
}
@Test
@Override
public void testUploaderName() throws Exception {
assertEquals("YouTube", extractor.getUploaderName());
}
@Test
@Override
public void testUploaderAvatarUrl() throws Exception {
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
ExtractorAsserts.assertContains("yt", uploaderAvatarUrl);
}
@Test
@Override
public void testStreamCount() throws Exception {
ExtractorAsserts.assertGreater(250, extractor.getStreamCount());
}
@Test
@Override
public void testUploaderVerified() throws Exception {
// YouTube doesn't provide this information for playlists
assertFalse(extractor.isUploaderVerified());
}
@Test
void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
}
@Test
void testDescription() throws ParsingException {
assertTrue(Utils.isNullOrEmpty(extractor.getDescription().getContent()));
}
}
public static class ContinuationsTests {
@BeforeAll

85
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_0.json Normal file
View File

@ -0,0 +1,85 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Referer": [
"https://www.youtube.com"
],
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Mon, 07 Aug 2023 17:06:40 GMT"
],
"expires": [
"Mon, 07 Aug 2023 17:06:40 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

79
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

387
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long