be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #262 from wb9688/pbj 

Improve yt_new
This commit is contained in:
Tobias Groza 2020-02-29 21:39:02 +01:00 committed by GitHub
parent fc465c8bf7 2807079531
commit 5a101fd17f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 605 additions and 554 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -2,33 +2,27 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/* /*
* Created by Christian Schabesberger on 25.07.16. * Created by Christian Schabesberger on 25.07.16.
@ -52,11 +46,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor { public class YoutubeChannelExtractor extends ChannelExtractor {
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc;
private JsonObject initialData; private JsonObject initialData;
private JsonObject videoTab;
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
@ -64,23 +55,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
final Response response = downloader.get(channelUrl, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
initialData = ajaxJson.getObject(1).getObject("response");
} }
@Override @Override
public String getNextPageUrl() throws ExtractionException { public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations")); if (getVideoTab() == null) return "";
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations"));
} }
@Nonnull @Nonnull
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
return CHANNEL_URL_BASE + getId(); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
} catch (ParsingException e) { } catch (ParsingException e) {
return super.getUrl(); return super.getUrl();
} }
@ -109,8 +104,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
try { try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
.getArray("thumbnails").getObject(0).getString("url"); .getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get avatar", e); throw new ParsingException("Could not get avatar", e);
} }
@ -127,17 +124,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null; return null;
} }
// the first characters of the banner URLs are different for each channel and some are not even valid URLs
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url; return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get banner", e); throw new ParsingException("Could not get banner", e);
} }
@ -157,13 +145,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) { if (subscriberInfo != null) {
try { try {
return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e); throw new ParsingException("Could not get subscriber count", e);
} }
} else { } else {
// If the element is null, the channel have the subscriber count disabled // If there's no subscribe button, the channel has the subscriber count disabled
return -1; if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
return -1;
} else {
return 0;
}
} }
} }
@ -181,8 +173,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents"); if (getVideoTab() != null) {
collectStreamsFrom(collector, videos); JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents")
.getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("gridRenderer").getArray("items");
collectStreamsFrom(collector, videos);
}
return new InfoItemsPage<>(collector, getNextPageUrl()); return new InfoItemsPage<>(collector, getNextPageUrl());
} }
@ -198,46 +194,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
fetchPage(); fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray ajaxJson; final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("sectionListContinuation"); .getObject("continuationContents").getObject("gridContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
} }
private String getNextPageUrlFrom(JsonArray continuations) { private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) { if (continuations == null) return "";
return "";
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation"); String continuation = nextContinuationData.getString("continuation");
@ -254,10 +223,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) { for (Object video : videos) {
JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
.getArray("contents").getObject(0); collector.commit(new YoutubeStreamInfoItemExtractor(
if (videoInfo.getObject("videoRenderer") != null) { ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) {
@Override @Override
public String getUploaderName() { public String getUploaderName() {
return uploaderName; return uploaderName;
@ -273,6 +241,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
private JsonObject getVideoTab() throws ParsingException { private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab;
JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs"); .getArray("tabs");
JsonObject videoTab = null; JsonObject videoTab = null;
@ -290,6 +260,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ParsingException("Could not find Videos tab"); throw new ParsingException("Could not find Videos tab");
} }
try {
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
} catch (Exception ignored) {}
this.videoTab = videoTab;
return videoTab; return videoTab;
} }
} }

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -7,8 +7,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/* /*
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
@ -41,15 +41,8 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
if (url.startsWith("//")) {
url = url.substring(2); return fixThumbnailUrl(url);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
@ -58,7 +51,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return channelInfoItem.getObject("title").getString("simpleText"); return getTextFromObject(channelInfoItem.getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
@ -67,7 +60,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work? String id = "channel/" + channelInfoItem.getString("channelId");
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get url", e); throw new ParsingException("Could not get url", e);
@ -77,7 +70,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
try { try {
String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0]; String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
return Utils.mixedNumberWordToLong(subscribers); return Utils.mixedNumberWordToLong(subscribers);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e); throw new ParsingException("Could not get subscriber count", e);
@ -87,8 +80,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
.getArray("runs").getObject(0).getString("text")));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get stream count", e); throw new ParsingException("Could not get stream count", e);
} }
@ -97,7 +89,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getDescription() throws ParsingException { public String getDescription() throws ParsingException {
try { try {
return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text"); return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get description", e); throw new ParsingException("Could not get description", e);
} }

86
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -2,37 +2,30 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc;
private JsonObject initialData; private JsonObject initialData;
private JsonObject uploaderInfo;
private JsonObject playlistInfo; private JsonObject playlistInfo;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@ -41,11 +34,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl(); final String url = getUrl() + "&pbj=1";
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
uploaderInfo = getUploaderInfo(); initialData = ajaxJson.getObject(1).getObject("response");
playlistInfo = getPlaylistInfo(); playlistInfo = getPlaylistInfo();
} }
@ -94,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text"); String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null) return name; if (name != null) return name;
} catch (Exception ignored) {} } catch (Exception ignored) {}
try { try {
@ -106,16 +99,23 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
String url = null;
try { try {
return playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {} } catch (Exception ignored) {}
try {
return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") if (url == null) {
.getArray("thumbnails").getObject(0).getString("url"); try {
} catch (Exception e) { url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
throw new ParsingException("Could not get playlist thumbnail", e); .getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get playlist thumbnail");
} }
return fixThumbnailUrl(url);
} }
@Override @Override
@ -127,8 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
return YoutubeChannelExtractor.CHANNEL_URL_BASE + return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
uploaderInfo.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader url", e); throw new ParsingException("Could not get playlist uploader url", e);
} }
@ -137,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return uploaderInfo.getObject("title").getArray("runs").getObject(0).getString("text"); return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name", e); throw new ParsingException("Could not get playlist uploader name", e);
} }
@ -146,7 +145,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
return uploaderInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e); throw new ParsingException("Could not get playlist uploader avatar", e);
} }
@ -155,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text"); String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get video count from playlist", e); throw new ParsingException("Could not get video count from playlist", e);
@ -184,32 +185,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray ajaxJson; final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation"); .getObject("continuationContents").getObject("playlistVideoListContinuation");

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private JsonObject playlistInfoItem; private JsonObject playlistInfoItem;
@ -17,8 +20,10 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
return playlistInfoItem.getArray("thumbnails").getObject(0).getArray("thumbnails") String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getObject(0).getString("url"); .getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
@ -27,7 +32,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return playlistInfoItem.getObject("title").getString("simpleText"); return getTextFromObject(playlistInfoItem.getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
@ -46,7 +51,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text"); return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get uploader name", e); throw new ParsingException("Could not get uploader name", e);
} }

68
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -2,30 +2,24 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/* /*
* Created by Christian Schabesberger on 22.07.2018 * Created by Christian Schabesberger on 22.07.2018
* *
@ -47,8 +41,6 @@ import javax.annotation.Nonnull;
*/ */
public class YoutubeSearchExtractor extends SearchExtractor { public class YoutubeSearchExtractor extends SearchExtractor {
private Document doc;
private JsonObject initialData; private JsonObject initialData;
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
@ -57,10 +49,11 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl(); final String url = getUrl() + "&pbj=1";
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
initialData = ajaxJson.getObject(1).getObject("response");
} }
@Nonnull @Nonnull
@ -79,8 +72,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
if (showingResultsForRenderer == null) { if (showingResultsForRenderer == null) {
return ""; return "";
} else { } else {
return showingResultsForRenderer.getObject("correctedQuery").getArray("runs") return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
.getObject(0).getString("text");
} }
} }
@ -88,11 +80,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override @Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException { public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector(); InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray videos = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
.getObject(0).getObject("itemSectionRenderer").getArray("contents");
for (Object section : sections) {
collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents"));
}
collectStreamsFrom(collector, videos);
return new InfoItemsPage<>(collector, getNextPageUrl()); return new InfoItemsPage<>(collector, getNextPageUrl());
} }
@ -110,33 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
} }
InfoItemsSearchCollector collector = getInfoItemSearchCollector(); InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray ajaxJson; final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation"); .getObject("continuationContents").getObject("itemSectionContinuation");
@ -153,8 +121,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
for (Object item : videos) { for (Object item : videos) {
if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") throw new NothingFoundException(getTextFromObject(((JsonObject) item)
.getObject("bodyText").getArray("runs").getObject(0).getString("text")); .getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (((JsonObject) item).getObject("videoRenderer") != null) { } else if (((JsonObject) item).getObject("videoRenderer") != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
} else if (((JsonObject) item).getObject("channelRenderer") != null) { } else if (((JsonObject) item).getObject("channelRenderer") != null) {

274
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,8 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
@ -13,8 +11,6 @@ import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -24,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Description;
@ -40,8 +37,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
@ -56,6 +51,11 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/* /*
* Created by Christian Schabesberger on 06.08.15. * Created by Christian Schabesberger on 06.08.15.
* *
@ -89,19 +89,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/ /*//////////////////////////////////////////////////////////////////////////*/
private Document doc; private JsonArray initialAjaxJson;
@Nullable @Nullable
private JsonObject playerArgs; private JsonObject playerArgs;
@Nonnull @Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>(); private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse; private JsonObject playerResponse;
private JsonObject initialData; private JsonObject initialData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit;
@Nonnull @Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>(); private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@ -115,16 +116,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getName() throws ParsingException { public String getName() throws ParsingException {
assertPageFetched(); assertPageFetched();
String title = null; String title = null;
try { try {
title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text"); title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (title == null) { if (title == null) {
try { try {
title = playerResponse.getObject("videoDetails").getString("title"); title = playerResponse.getObject("videoDetails").getString("title");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (title == null) throw new ParsingException("Could not get name");
} }
if (title != null) return title;
throw new ParsingException("Could not get name"); return title;
} }
@Override @Override
@ -144,8 +149,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (Exception ignored) {} } catch (Exception ignored) {}
try { try {
if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
@ -163,7 +168,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { try {
// TODO this parses English formatted dates only, we need a better approach to parse the textual date // TODO this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText")); getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d); return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {} } catch (Exception ignored) {}
throw new ParsingException("Could not get upload date"); throw new ParsingException("Could not get upload date");
@ -187,8 +192,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
return thumbnails.getObject(thumbnails.size() - 1).getString("url"); String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url"); throw new ParsingException("Could not get thumbnail url");
} }
@ -201,55 +207,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
// description with more info on links // description with more info on links
try { try {
boolean htmlConversionRequired = false; String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); return new Description(description, Description.HTML);
StringBuilder descriptionBuilder = new StringBuilder(descriptions.size());
for (Object textObjectHolder : descriptions) {
JsonObject textHolder = (JsonObject) textObjectHolder;
String text = textHolder.getString("text");
if (textHolder.getObject("navigationEndpoint") != null) {
// The text is a link. Get the URL it points to and generate a HTML link of it
if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) {
String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
if (url != null && !url.isEmpty()) {
descriptionBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
} else {
descriptionBuilder.append(text);
}
break;
}
}
} else if (internUrl.startsWith("http")) {
descriptionBuilder.append("<a href=\"").append(internUrl).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
}
continue;
}
continue;
}
if (text != null) {
descriptionBuilder.append(text);
}
}
String description = descriptionBuilder.toString();
if (!description.isEmpty()) {
if (htmlConversionRequired) {
description = description.replaceAll("\\n", "<br>");
description = description.replaceAll(" ", " &nbsp;");
return new Description(description, Description.HTML);
}
return new Description(description, Description.PLAIN_TEXT);
}
} catch (Exception ignored) { } } catch (Exception ignored) { }
// raw non-html description // raw non-html description
@ -261,17 +220,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public int getAgeLimit() throws ParsingException { public int getAgeLimit() {
assertPageFetched(); if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet");
if (!isAgeRestricted) {
return NO_AGE_LIMIT; return ageLimit;
}
try {
return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]")
.attr(CONTENT).replace("+", ""));
} catch (Exception e) {
throw new ParsingException("Could not get age restriction");
}
} }
@Override @Override
@ -311,24 +263,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public long getViewCount() throws ParsingException { public long getViewCount() throws ParsingException {
assertPageFetched(); assertPageFetched();
String views = null; String views = null;
try { try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount") views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount"));
.getArray("runs").getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (views == null) {
try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText");
} catch (Exception ignored) {}
}
if (views == null) { if (views == null) {
try { try {
views = playerResponse.getObject("videoDetails").getString("viewCount"); views = playerResponse.getObject("videoDetails").getString("viewCount");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (views == null) throw new ParsingException("Could not get view count");
} }
if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views));
throw new ParsingException("Could not get view count"); return Long.parseLong(Utils.removeNonDigitCharacters(views));
} }
@Override @Override
@ -381,17 +330,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
String uploaderId = null;
try { try {
uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (uploaderUrl != null) return uploaderUrl;
} catch (Exception ignored) {}
try {
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null)
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (uploaderId == null) {
try {
uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
} catch (Exception ignored) {}
}
if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId;
throw new ParsingException("Could not get uploader url"); throw new ParsingException("Could not get uploader url");
} }
@ -400,44 +348,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
assertPageFetched(); assertPageFetched();
String uploaderName = null; String uploaderName = null;
try { try {
uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("title").getArray("runs").getObject(0).getString("text"); .getObject("videoOwnerRenderer").getObject("title"));
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (uploaderName == null) { if (uploaderName == null) {
try { try {
uploaderName = playerResponse.getObject("videoDetails").getString("author"); uploaderName = playerResponse.getObject("videoDetails").getString("author");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (uploaderName == null) throw new ParsingException("Could not get uploader name");
} }
if (uploaderName != null) return uploaderName;
throw new ParsingException("Could not get uploader name"); return uploaderName;
} }
@Nonnull @Nonnull
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
String uploaderAvatarUrl = null;
try { try {
uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer")
.getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail")
.getArray("thumbnails").getObject(0).getString("url");
if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) {
return uploaderAvatarUrl;
}
} catch (Exception ignored) {}
try {
uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (uploaderAvatarUrl == null) { return fixThumbnailUrl(url);
throw new ParsingException("Could not get uploader avatar url"); } catch (Exception e) {
throw new ParsingException("Could not get uploader avatar url", e);
} }
return uploaderAvatarUrl;
} }
@Nonnull @Nonnull
@ -578,9 +517,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public StreamInfoItem getNextStream() throws ExtractionException { public StreamInfoItem getNextStream() throws ExtractionException {
assertPageFetched(); assertPageFetched();
if (isAgeRestricted) {
return null; if (getAgeLimit() != NO_AGE_LIMIT) return null;
}
try { try {
final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("secondaryResults").getObject("secondaryResults").getArray("results") .getObject("secondaryResults").getObject("secondaryResults").getArray("results")
@ -599,9 +538,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
assertPageFetched(); assertPageFetched();
if (isAgeRestricted) {
return null; if (getAgeLimit() != NO_AGE_LIMIT) return null;
}
try { try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
@ -625,23 +564,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
*/ */
@Override @Override
public String getErrorMessage() { public String getErrorMessage() {
StringBuilder errorReason; return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
if (errorElement == null) {
errorReason = null;
} else {
String errorMessage = errorElement.text();
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
}
return errorReason != null ? errorReason.toString() : "";
} }
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
@ -651,11 +575,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats"; private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:"; private static final String HTTPS = "https:";
private static final String CONTENT = "content";
private static final String DECRYPTION_FUNC_NAME = "decrypt"; private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
@ -667,32 +588,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private volatile String decryptionCode = ""; private volatile String decryptionCode = "";
private String pageHtml = null;
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; final String url = getUrl() + "&pbj=1";
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
pageHtml = response.responseBody(); initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
final String playerUrl; final String playerUrl;
// Check if the video is age restricted
if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
initialData = initialAjaxJson.getObject(2).getObject("response");
ageLimit = 18;
final EmbeddedInfo info = getEmbeddedInfo(); final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url; playerUrl = info.url;
isAgeRestricted = true;
} else { } else {
final JsonObject ytPlayerConfig = getPlayerConfig(); initialData = initialAjaxJson.getObject(3).getObject("response");
playerArgs = getPlayerArgs(ytPlayerConfig); ageLimit = NO_AGE_LIMIT;
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false; playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
} }
playerResponse = getPlayerResponse(); playerResponse = getPlayerResponse();
initialData = YoutubeParsingHelper.getInitialData(pageHtml);
if (decryptionCode.isEmpty()) { if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl); decryptionCode = loadDecryptionCode(playerUrl);
@ -703,21 +624,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
private JsonObject getPlayerConfig() throws ParsingException {
try {
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
return JsonParser.object().from(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
if (errorReason.isEmpty()) {
throw new ContentNotAvailableException("Content not available: player config empty", e);
}
throw new ContentNotAvailableException("Content not available", e);
} catch (Exception e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
JsonObject playerArgs; JsonObject playerArgs;
@ -869,7 +775,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() { private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
// If the video is age restricted getPlayerConfig will fail // If the video is age restricted getPlayerConfig will fail
if (isAgeRestricted) return Collections.emptyList(); if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
final JsonObject captions; final JsonObject captions;
if (!playerResponse.has("captions")) { if (!playerResponse.has("captions")) {
@ -939,6 +845,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"); .getObject("results").getObject("results").getArray("contents");
JsonObject videoPrimaryInfoRenderer = null; JsonObject videoPrimaryInfoRenderer = null;
@ -954,10 +862,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoPrimaryInfoRenderer"); throw new ParsingException("Could not find videoPrimaryInfoRenderer");
} }
this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer;
return videoPrimaryInfoRenderer; return videoPrimaryInfoRenderer;
} }
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"); .getObject("results").getObject("results").getArray("contents");
JsonObject videoSecondaryInfoRenderer = null; JsonObject videoSecondaryInfoRenderer = null;
@ -973,6 +884,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoSecondaryInfoRenderer"); throw new ParsingException("Could not find videoSecondaryInfoRenderer");
} }
this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer;
return videoSecondaryInfoRenderer; return videoSecondaryInfoRenderer;
} }
@ -1010,9 +922,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
urlAndItags.put(streamUrl, itagItem); urlAndItags.put(streamUrl, itagItem);
} }
} catch (UnsupportedEncodingException ignored) { } catch (UnsupportedEncodingException ignored) {}
}
} }
} }
@ -1023,17 +933,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public List<Frameset> getFrames() throws ExtractionException { public List<Frameset> getFrames() throws ExtractionException {
try { try {
final String script = doc.select("#player-api").first().siblingElements().select("script").html(); JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
int p = script.indexOf("ytplayer.config");
if (p == -1) {
return Collections.emptyList();
}
p = script.indexOf('{', p);
int e = script.indexOf("ytplayer.load", p);
if (e == -1) {
return Collections.emptyList();
}
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
final String resp = jo.getObject("args").getString("player_response"); final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp); jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");

121
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -6,7 +6,6 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -15,6 +14,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/* /*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe. * YoutubeStreamInfoItemExtractor.java is part of NewPipe.
@ -76,89 +79,95 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
String name = null; String name = getTextFromObject(videoInfo.getObject("title"));
try {
name = videoInfo.getObject("title").getString("simpleText");
} catch (Exception ignored) {}
if (name == null) {
try {
name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text");
} catch (Exception ignored) {}
}
if (name != null && !name.isEmpty()) return name; if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get name"); throw new ParsingException("Could not get name");
} }
@Override @Override
public long getDuration() throws ParsingException { public long getDuration() throws ParsingException {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
String duration = null;
try { try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1; duration = getTextFromObject(videoInfo.getObject("lengthText"));
return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); } catch (Exception ignored) {}
} catch (Exception e) {
throw new ParsingException("Could not get duration", e); if (duration == null) {
try {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
}
} catch (Exception ignored) {}
if (duration == null) throw new ParsingException("Could not get duration");
} }
return YoutubeParsingHelper.parseDurationString(duration);
} }
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
String name = null; String name = null;
try { try {
name = videoInfo.getObject("longBylineText").getArray("runs") name = getTextFromObject(videoInfo.getObject("longBylineText"));
.getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (name == null) { if (name == null) {
try { try {
name = videoInfo.getObject("ownerText").getArray("runs") name = getTextFromObject(videoInfo.getObject("ownerText"));
.getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
} catch (Exception ignored) {}
if (name == null) throw new ParsingException("Could not get uploader name");
}
} }
if (name == null) {
try { return name;
name = videoInfo.getObject("shortBylineText").getArray("runs")
.getObject(0).getString("text");
} catch (Exception ignored) {}
}
if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get uploader name");
} }
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
String url = null;
try { try {
String id = null; url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try { try {
id = videoInfo.getObject("longBylineText").getArray("runs") url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
.getObject(0).getObject("navigationEndpoint") .getArray("runs").getObject(0).getObject("navigationEndpoint"));
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (id == null) {
if (url == null) {
try { try {
id = videoInfo.getObject("ownerText").getArray("runs") url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getObject(0).getObject("navigationEndpoint") .getArray("runs").getObject(0).getObject("navigationEndpoint"));
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get uploader url");
} }
if (id == null) {
try {
id = videoInfo.getObject("shortBylineText").getArray("runs")
.getObject(0).getObject("navigationEndpoint")
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {}
}
if (id == null || id.isEmpty()) {
throw new IllegalArgumentException("is empty");
}
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) {
throw new ParsingException("Could not get uploader url");
} }
return url;
} }
@Nullable @Nullable
@Override @Override
public String getTextualUploadDate() { public String getTextualUploadDate() {
try { try {
return videoInfo.getObject("publishedTimeText").getString("simpleText"); return getTextFromObject(videoInfo.getObject("publishedTimeText"));
} catch (Exception e) { } catch (Exception e) {
// upload date is not always available, e.g. in playlists // upload date is not always available, e.g. in playlists
return null; return null;
@ -185,15 +194,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
return -1; return -1;
} }
String viewCount; String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
if (getStreamType() == StreamType.LIVE_STREAM) {
viewCount = videoInfo.getObject("viewCountText")
.getArray("runs").getObject(0).getString("text");
} else {
viewCount = videoInfo.getObject("viewCountText").getString("simpleText");
}
if (viewCount.equals("Recommended for you")) return -1;
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
} catch (NumberFormatException e) {
return -1;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get view count", e); throw new ParsingException("Could not get view count", e);
} }
@ -203,8 +208,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
// TODO: Don't simply get the first item, but look at all thumbnails and their resolution // TODO: Don't simply get the first item, but look at all thumbnails and their resolution
return videoInfo.getObject("thumbnail").getArray("thumbnails") String url = videoInfo.getObject("thumbnail").getArray("thumbnails")
.getObject(0).getString("url"); .getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -25,13 +25,11 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -39,6 +37,9 @@ import java.io.IOException;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> { public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData; private JsonObject initialData;
@ -50,11 +51,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + final String url = getUrl() + "?pbj=1&gl="
"?gl=" + getExtractorContentCountry().getCountryCode(); + getExtractorContentCountry().getCountryCode();
final Response response = downloader.get(url, getExtractorLocalization()); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
initialData = ajaxJson.getObject(1).getObject("response");
} }
@Override @Override
@ -72,8 +74,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public String getName() throws ParsingException { public String getName() throws ParsingException {
String name; String name;
try { try {
name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title") name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
.getArray("runs").getObject(0).getString("text");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get Trending name", e); throw new ParsingException("Could not get Trending name", e);
} }
@ -87,17 +88,21 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer").getArray("items");
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents");
for (Object ul : firstPageElements) { for (Object itemSectionRenderer : itemSectionRenderers) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");
if (expandedShelfContentsRenderer != null) {
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
}
} }
return new InfoItemsPage<>(collector, getNextPageUrl()); return new InfoItemsPage<>(collector, getNextPageUrl());

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View File

@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
return instance; return instance;
} }
/**
* Returns URL to channel from an ID
*
* @param id Channel ID including e.g. 'channel/'
* @param contentFilters
* @param searchFilter
* @return URL to channel
*/
@Override @Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) { public String getUrl(String id, List<String> contentFilters, String searchFilter) {
return "https://www.youtube.com/" + id; return "https://www.youtube.com/" + id;

229
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -5,18 +5,34 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
/* /*
* Created by Christian Schabesberger on 02.03.16. * Created by Christian Schabesberger on 02.03.16.
@ -43,7 +59,9 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() { private YoutubeParsingHelper() {
} }
public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@ -160,58 +178,189 @@ public class YoutubeParsingHelper {
} }
} }
public static boolean isHardcodedClientVersionValid() throws IOException {
try {
final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers).responseBody();
if (response.length() > 50) { // ensure to have a valid response
return true;
}
} catch (ReCaptchaException ignored) {}
return false;
}
/** /**
* Get the client version from a page * Get the client version from a page
* @param initialData
* @param html The page HTML
* @return * @return
* @throws ParsingException * @throws ParsingException
*/ */
public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { public static String getClientVersion() throws ParsingException, IOException {
if (initialData == null) initialData = getInitialData(html); if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first if (isHardcodedClientVersionValid()) {
for (Object service : serviceTrackingParams) { clientVersion = HARDCODED_CLIENT_VERSION;
JsonObject s = (JsonObject) service; return clientVersion;
if (s.getString("service").equals("CSI")) { }
JsonArray params = s.getArray("params");
for (Object param: params) { // Try extracting it from YouTube's website otherwise
JsonObject p = (JsonObject) param; try {
String key = p.getString("key"); final String url = "https://www.youtube.com/results?search_query=test";
if (key != null && key.equals("cver")) { final String html = getDownloader().get(url).responseBody();
return p.getString("value"); JsonObject initialData = getInitialData(html);
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first
for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("cver")) {
clientVersion = p.getString("value");
return clientVersion;
}
} }
} } else if (s.getString("service").equals("ECATCHER")) {
} else if (s.getString("service").equals("ECATCHER")) { // fallback to get a shortened client version which does not contain the last two digits
// fallback to get a shortened client version which does not contain the last do digits JsonArray params = s.getArray("params");
JsonArray params = s.getArray("params"); for (Object param : params) {
for (Object param: params) { JsonObject p = (JsonObject) param;
JsonObject p = (JsonObject) param; String key = p.getString("key");
String key = p.getString("key"); if (key != null && key.equals("client.version")) {
if (key != null && key.equals("client.version")) { shortClientVersion = p.getString("value");
shortClientVersion = p.getString("value"); }
} }
} }
} }
}
String clientVersion; String contextClientVersion;
String[] patterns = { String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"", "innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)" "client.version=([0-9\\.]+)"
}; };
for (String pattern: patterns) { for (String pattern : patterns) {
try { try {
clientVersion = Parser.matchGroup1(pattern, html); contextClientVersion = Parser.matchGroup1(pattern, html);
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
} catch (Exception ignored) {} clientVersion = contextClientVersion;
} return clientVersion;
}
} catch (Exception ignored) {
}
}
if (shortClientVersion != null) return shortClientVersion; if (shortClientVersion != null) {
clientVersion = shortClientVersion;
return clientVersion;
}
} catch (Exception ignored) {}
throw new ParsingException("Could not get client version"); throw new ParsingException("Could not get client version");
} }
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
return null;
}
return url;
}
}
} else if (internUrl.startsWith("http")) {
return internUrl;
}
} else if (navigationEndpoint.getObject("browseEndpoint") != null) {
return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl");
} else if (navigationEndpoint.getObject("watchEndpoint") != null) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
}
return null;
}
public static String getTextFromObject(JsonObject textObject, boolean html) {
if (textObject.has("simpleText")) return textObject.getString("simpleText");
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
continue;
}
}
textBuilder.append(text);
}
String text = textBuilder.toString();
if (html) {
text = text.replaceAll("\\n", "<br>");
text = text.replaceAll(" ", " &nbsp;");
}
return text;
}
public static String getTextFromObject(JsonObject textObject) {
return getTextFromObject(textObject, false);
}
public static String fixThumbnailUrl(String thumbnailUrl) {
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = thumbnailUrl.substring(2);
}
if (thumbnailUrl.startsWith(HTTP)) {
thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl);
} else if (!thumbnailUrl.startsWith(HTTPS)) {
thumbnailUrl = "https://" + thumbnailUrl;
}
return thumbnailUrl;
}
public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final String response = getDownloader().get(url, headers, localization).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("JSON response is too short");
}
try {
return JsonParser.array().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
} }

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java Normal file
View File

@ -0,0 +1,24 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
public class YoutubeParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testIsHardcodedClientVersionValid() throws IOException {
assertTrue("Hardcoded client version is not valid anymore",
YoutubeParsingHelper.isHardcodedClientVersionValid());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest {
@Test @Test
public void testUploaderUrl() throws Exception { public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl());
} }
@Test @Test

46
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View File

@ -12,10 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import java.util.regex.Pattern; import java.net.URL;
import java.net.URLDecoder;
import java.util.LinkedHashMap;
import java.util.Map;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
import static org.junit.Assert.*; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest {
@ -47,18 +54,26 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
} }
} }
assertFalse("First and second page are equal", equals); assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl());
} }
@Test @Test
public void testGetSecondPageUrl() throws Exception { public void testGetSecondPageUrl() throws Exception {
// check that ctoken, continuation and itct are longer than 5 characters URL url = new URL(extractor.getNextPageUrl());
Pattern pattern = Pattern.compile(
"https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1" assertEquals(url.getHost(), "www.youtube.com");
+ "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?" assertEquals(url.getPath(), "/results");
);
assertTrue(pattern.matcher(extractor.getNextPageUrl()).find()); Map<String, String> queryPairs = new LinkedHashMap<>();
for (String queryPair : url.getQuery().split("&")) {
int index = queryPair.indexOf("=");
queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"),
URLDecoder.decode(queryPair.substring(index + 1), "UTF-8"));
}
assertEquals("pewdiepie", queryPairs.get("search_query"));
assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation"));
assertTrue(queryPairs.get("continuation").length() > 5);
assertTrue(queryPairs.get("itct").length() > 5);
} }
@Ignore @Ignore
@ -77,13 +92,18 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
if (item instanceof ChannelInfoItem) { if (item instanceof ChannelInfoItem) {
ChannelInfoItem channel = (ChannelInfoItem) item; ChannelInfoItem channel = (ChannelInfoItem) item;
if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie
assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl());
} else { break;
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
} }
} }
} }
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof ChannelInfoItem) {
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
}
}
} }
@Test @Test

30
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java
View File

@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.net.URL;
import java.net.URLDecoder;
import java.util.LinkedHashMap;
import java.util.Map;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
@Test @Test
public void testGetUrl() throws Exception { public void testGetUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl());
} }
@Test @Test
public void testGetSecondPageUrl() throws Exception { public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); URL url = new URL(extractor.getNextPageUrl());
assertEquals(url.getHost(), "www.youtube.com");
assertEquals(url.getPath(), "/results");
Map<String, String> queryPairs = new LinkedHashMap<>();
for (String queryPair : url.getQuery().split("&")) {
int index = queryPair.indexOf("=");
queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"),
URLDecoder.decode(queryPair.substring(index + 1), "UTF-8"));
}
assertEquals("pewdiepie", queryPairs.get("search_query"));
assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation"));
assertTrue(queryPairs.get("continuation").length() > 5);
assertTrue(queryPairs.get("itct").length() > 5);
} }
@Test @Test
@ -101,14 +121,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
} }
} }
assertFalse("First and second page are equal", equals); assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl());
} }
@Test @Test
public void testSuggestionNotNull() throws Exception { public void testSuggestionNotNull() {
//todo write a real test //todo write a real test
assertTrue(extractor.getSearchSuggestion() != null); assertNotNull(extractor.getSearchSuggestion());
} }

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View File

@ -11,11 +11,11 @@ public class YoutubeSearchQHTest {
@Test @Test
public void testRegularValues() throws Exception { public void testRegularValues() throws Exception {
assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
} }
@Test @Test

42
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -10,17 +10,24 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.List; import java.util.List;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -89,7 +96,6 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi..."));
} }
@Test @Test
@ -142,12 +148,12 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetAudioStreams() throws IOException, ExtractionException { public void testGetAudioStreams() throws ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty()); assertFalse(extractor.getAudioStreams().isEmpty());
} }
@Test @Test
public void testGetVideoStreams() throws IOException, ExtractionException { public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) { for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url); assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0); assertTrue(s.resolution.length() > 0);
@ -169,7 +175,7 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors()); Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty()); assertFalse(relatedVideos.getItems().isEmpty());
@ -177,13 +183,13 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException { public void testGetSubtitlesListDefault() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty()); assertTrue(extractor.getSubtitlesDefault().isEmpty());
} }
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
} }
@ -223,10 +229,6 @@ public class YoutubeStreamExtractorDefaultTest {
assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub..."));
assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8..."));
assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/..."));
} }
} }
@ -249,15 +251,11 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); final String description = extractor.getDescription().getContent();
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
} }
} }

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.*; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest {
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A");
extractor.fetchPage(); extractor.fetchPage();
} }
@ -49,8 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest {
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists"));
assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba..."));
} }
@Test @Test
@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest {
} }
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors()); Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty()); assertFalse(relatedVideos.getItems().isEmpty());
@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest {
} }
@Test @Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException { public void testGetSubtitlesListDefault() {
assertTrue(extractor.getSubtitlesDefault().isEmpty()); assertTrue(extractor.getSubtitlesDefault().isEmpty());
} }
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() {
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
} }
} }