Merge pull request #332 from wb9688/learning-playlist
Support YouTube's learning playlists
This commit is contained in:
Tobias Groza
GitHub
commit
bdb0f2ae6b
|
|
@ -8,16 +8,21 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
|
|||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
||||
|
|
@ -27,6 +32,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
|||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
private JsonArray initialAjaxJson;
|
||||
private JsonObject initialData;
|
||||
private JsonObject playlistInfo;
|
||||
|
||||
|
|
@ -38,9 +44,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
final String url = getUrl() + "&pbj=1";
|
||||
|
||||
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
|
||||
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
|
||||
|
||||
initialData = ajaxJson.getObject(1).getObject("response");
|
||||
initialData = initialAjaxJson.getObject(1).getObject("response");
|
||||
YoutubeParsingHelper.defaultAlertsCheck(initialData);
|
||||
|
||||
playlistInfo = getPlaylistInfo();
|
||||
|
|
@ -152,34 +158,47 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getSubChannelName() throws ParsingException {
|
||||
public String getSubChannelName() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getSubChannelUrl() throws ParsingException {
|
||||
public String getSubChannelUrl() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getSubChannelAvatarUrl() throws ParsingException {
|
||||
public String getSubChannelAvatarUrl() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getInitialPage() {
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
|
||||
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
|
||||
.getObject("sectionListRenderer").getArray("contents").getObject(0)
|
||||
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
|
||||
.getObject("playlistVideoListRenderer").getArray("contents");
|
||||
.getObject("itemSectionRenderer").getArray("contents");
|
||||
|
||||
if (contents.getObject(0).has("playlistSegmentRenderer")) {
|
||||
for (final Object segment : contents) {
|
||||
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
|
||||
collectTrailerFrom(collector, ((JsonObject) segment));
|
||||
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
|
||||
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
|
||||
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
|
||||
}
|
||||
}
|
||||
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
|
||||
final JsonArray videos = contents.getObject(0)
|
||||
.getObject("playlistVideoListRenderer").getArray("contents");
|
||||
collectStreamsFrom(collector, videos);
|
||||
}
|
||||
|
||||
collectStreamsFrom(collector, videos);
|
||||
return new InfoItemsPage<>(collector, getNextPageUrl());
|
||||
}
|
||||
|
||||
|
|
@ -189,10 +208,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
|
||||
|
||||
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
|
||||
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
|
||||
.getObject("continuationContents").getObject("playlistVideoListContinuation");
|
||||
|
||||
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
|
||||
|
|
@ -200,7 +219,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
|
||||
}
|
||||
|
||||
private String getNextPageUrlFrom(JsonArray continuations) {
|
||||
private String getNextPageUrlFrom(final JsonArray continuations) {
|
||||
if (isNullOrEmpty(continuations)) {
|
||||
return "";
|
||||
}
|
||||
|
|
@ -212,9 +231,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
+ "&itct=" + clickTrackingParams;
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
|
||||
collector.reset();
|
||||
|
||||
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
for (Object video : videos) {
|
||||
|
|
@ -228,4 +245,76 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void collectTrailerFrom(final StreamInfoItemsCollector collector,
|
||||
final JsonObject segment) {
|
||||
collector.commit(new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
|
||||
.getObject("title"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
return YoutubeStreamLinkHandlerFactory.getInstance()
|
||||
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
|
||||
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
|
||||
.getUrl();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
|
||||
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
||||
return fixThumbnailUrl(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamType getStreamType() {
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAd() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDuration() throws ParsingException {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
getTextFromObject(segment.getObject("playlistSegmentRenderer")
|
||||
.getObject("segmentAnnotation")).split("•")[0]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
return YoutubePlaylistExtractor.this.getUploaderName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
return YoutubePlaylistExtractor.this.getUploaderUrl();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public String getTextualUploadDate() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public DateWrapper getUploadDate() {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -254,4 +254,102 @@ public class YoutubePlaylistExtractorTest {
|
|||
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
|
||||
}
|
||||
}
|
||||
|
||||
public static class LearningPlaylist implements BasePlaylistExtractorTest {
|
||||
private static YoutubePlaylistExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = (YoutubePlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Extractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testServiceId() {
|
||||
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testName() throws Exception {
|
||||
String name = extractor.getName();
|
||||
assertTrue(name, name.startsWith("Anatomy & Physiology"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ListExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testRelatedItems() throws Exception {
|
||||
defaultTestRelatedItems(extractor);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
defaultTestMoreItems(extractor);
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// PlaylistExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testThumbnailUrl() throws Exception {
|
||||
final String thumbnailUrl = extractor.getThumbnailUrl();
|
||||
assertIsSecureUrl(thumbnailUrl);
|
||||
assertTrue(thumbnailUrl, thumbnailUrl.contains("yt"));
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testBannerUrl() throws Exception {
|
||||
final String bannerUrl = extractor.getBannerUrl();
|
||||
assertIsSecureUrl(bannerUrl);
|
||||
assertTrue(bannerUrl, bannerUrl.contains("yt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploaderUrl() throws Exception {
|
||||
assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploaderName() throws Exception {
|
||||
final String uploaderName = extractor.getUploaderName();
|
||||
assertTrue(uploaderName, uploaderName.contains("CrashCourse"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploaderAvatarUrl() throws Exception {
|
||||
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
|
||||
assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStreamCount() throws Exception {
|
||||
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue