Merge pull request #332 from wb9688/learning-playlist

Support YouTube's learning playlists
This commit is contained in:
Tobias Groza 2020-05-29 10:11:08 +02:00 committed by GitHub
commit bdb0f2ae6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 203 additions and 16 deletions

View File

@ -8,16 +8,21 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
@ -27,6 +32,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonArray initialAjaxJson;
private JsonObject initialData;
private JsonObject playlistInfo;
@ -38,9 +44,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
initialData = initialAjaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo();
@ -152,34 +158,47 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
.getObject("itemSectionRenderer").getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
}
}
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonArray videos = contents.getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
collectStreamsFrom(collector, videos);
}
collectStreamsFrom(collector, videos);
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@ -189,10 +208,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
@ -200,7 +219,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
}
private String getNextPageUrlFrom(JsonArray continuations) {
private String getNextPageUrlFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return "";
}
@ -212,9 +231,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
+ "&itct=" + clickTrackingParams;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
collector.reset();
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
@ -228,4 +245,76 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
}
private void collectTrailerFrom(final StreamInfoItemsCollector collector,
final JsonObject segment) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public String getName() throws ParsingException {
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("title"));
}
@Override
public String getUrl() throws ParsingException {
return YoutubeStreamLinkHandlerFactory.getInstance()
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
.getUrl();
}
@Override
public String getThumbnailUrl() {
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
}
@Override
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
@Override
public long getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("segmentAnnotation")).split("")[0]);
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderName() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderUrl();
}
@Nullable
@Override
public String getTextualUploadDate() {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
});
}
}

View File

@ -254,4 +254,102 @@ public class YoutubePlaylistExtractorTest {
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
}
}
public static class LearningPlaylist implements BasePlaylistExtractorTest {
private static YoutubePlaylistExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
String name = extractor.getName();
assertTrue(name, name.startsWith("Anatomy & Physiology"));
}
@Test
public void testId() throws Exception {
assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertTrue(thumbnailUrl, thumbnailUrl.contains("yt"));
}
@Ignore
@Test
public void testBannerUrl() throws Exception {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
}
@Test
public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl());
}
@Test
public void testUploaderName() throws Exception {
final String uploaderName = extractor.getUploaderName();
assertTrue(uploaderName, uploaderName.contains("CrashCourse"));
}
@Test
public void testUploaderAvatarUrl() throws Exception {
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt"));
}
@Test
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
}
}
}