be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[Youtube] apply wb9688 suggestion (mix) 

Channel mix adjusments and test
Don't accept youtube music mix urls as playlist
Don't override playlistData to keep getInitialPage()
Remove json constants
Indentation
This commit is contained in:
Xiang Rong Lin 2020-03-21 18:48:12 +01:00 committed by XiangRongLin
parent 822cf307f7
commit 3ff8619bcc
5 changed files with 64 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -193,13 +193,23 @@ public class YoutubeParsingHelper {
}
/**
* Checks if the given playlist id is a mix (auto-generated playlist)
* Ids from a mix start with "RD"
* Checks if the given playlist id is a youtube mix (auto-generated playlist)
* Ids from a youtube mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a mix
* @return Whether given id belongs to a youtube mix
*/
public static boolean isYoutubeMixId(String playlistId) {
return playlistId.startsWith("RD");
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
}
/**
* Checks if the given playlist id is a youtube music mix (auto-generated playlist)
* Ids from a youtube music mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a youtube music mix
*/
public static boolean isYoutubeMusicMixId(String playlistId) {
return playlistId.startsWith("RDAMVM");
}
public static JsonObject getInitialData(String html) throws ParsingException {
@ -427,9 +437,9 @@ public class YoutubeParsingHelper {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
@ -457,6 +467,7 @@ public class YoutubeParsingHelper {
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
url = url.replaceAll("&", "&");
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
continue;
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -144,7 +144,7 @@ public class YoutubeService extends StreamingService {
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
}

67
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -25,13 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
*/
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
private final static String CONTENTS = "contents";
private final static String RESPONSE = "response";
private final static String PLAYLIST = "playlist";
private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults";
private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer";
private JsonObject initialData;
private JsonObject playlistData;
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@ -43,9 +38,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE);
playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS)
.getObject(PLAYLIST).getObject(PLAYLIST);
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
}
@Nonnull
@ -62,7 +57,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
public String getThumbnailUrl() throws ParsingException {
try {
final String playlistId = playlistData.getString("playlistId");
return getThumbnailUrlFromId(playlistId);
try {
return getThumbnailUrlFromPlaylistId(playlistId);
} catch (ParsingException e) {
//fallback to thumbnail of current video. Always the case for channel mix
return getThumbnailUrlFromVideoId(
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
.getString("videoId"));
}
} catch (Exception e) {
throw new ParsingException("Could not get playlist thumbnail", e);
}
@ -101,20 +103,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray(CONTENTS));
collectStreamsFrom(collector, playlistData.getArray("contents"));
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public String getNextPageUrl() throws ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS)
.get(playlistData.getArray(CONTENTS).size() - 1));
if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) {
return getNextPageUrlFrom(playlistData);
}
private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
.get(playlistData.getArray("contents").size() - 1));
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
throw new ExtractionException("Could not extract next page url");
}
return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER)
.getObject("navigationEndpoint").getObject("commandMetadata")
.getObject("webCommandMetadata").getString("url") + "&pbj=1";
//Index of video in mix is missing, but adding it doesn't appear to have any effect.
//And since the index needs to be tracked by us, it is left out
return getUrlFromNavigationEndpoint(
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+ "&pbj=1";
}
@Override
@ -127,21 +135,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
playlistData =
ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS)
.getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST)
.getObject(PLAYLIST);
final JsonArray streams = playlistData.getArray(CONTENTS);
JsonObject playlistData =
ajaxJson.getObject(3).getObject("response").getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist")
.getObject("playlist");
final JsonArray streams = playlistData.getArray("contents");
//Because continuation requests are created with the last video of previous request as start
streams.remove(0);
collectStreamsFrom(collector, streams);
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData));
}
private void collectStreamsFrom(
@Nonnull StreamInfoItemsCollector collector,
@Nullable JsonArray streams) {
collector.reset();
if (streams == null) {
return;
@ -152,7 +159,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
for (Object stream : streams) {
if (stream instanceof JsonObject) {
JsonObject streamInfo = ((JsonObject) stream)
.getObject(PLAYLIST_PANEL_VIDEO_RENDERER);
.getObject("playlistPanelVideoRenderer");
if (streamInfo != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
}
@ -160,16 +167,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
}
}
private String getThumbnailUrlFromId(String playlistId) throws ParsingException {
private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException {
final String videoId;
if (playlistId.startsWith("RDMM")) {
videoId = playlistId.substring(4);
} else if (playlistId.startsWith("RDCMUC")) {
throw new ParsingException("is channel mix");
} else {
videoId = playlistId.substring(2);
}
if (videoId.isEmpty()) {
throw new ParsingException("videoId is empty");
}
return getThumbnailUrlFromVideoId(videoId);
}
private String getThumbnailUrlFromVideoId(String videoId) {
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -64,11 +64,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public boolean onAcceptUrl(final String url) {
try {
getId(url);
String playlistId = getId(url);
//Because youtube music mix are not supported yet.
return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId);
} catch (ParsingException e) {
return false;
}
return true;
}
/**

19
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -93,11 +94,6 @@ public class YoutubeMixPlaylistExtractorTest {
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
public static class MixWithIndex {
@ -166,11 +162,6 @@ public class YoutubeMixPlaylistExtractorTest {
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
@ -264,12 +255,13 @@ public class YoutubeMixPlaylistExtractorTest {
extractor.getPage("");
}
@Test(expected = NullPointerException.class)
@Test(expected = ExtractionException.class)
public void invalidVideoId() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
extractor.fetchPage();
extractor.getName();
}
}
@ -329,10 +321,5 @@ public class YoutubeMixPlaylistExtractorTest {
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
}