[YouTube] Fix hashtags links extraction and escape text in attribute descriptions + HTML links
webCommandMetadata object is contained inside a commandMetadata one, so it is not accessible from the root of the navigationEndpoint object. The corresponding statement has been moved at the bottom of the specific endpoints parsing, as the webCommandMetadata object is present almost everywhere, otherwise URLs of some endpoints would have be changed, such as uploader URLs (from channel IDs to handles). As no ParsingException is now thrown by getUrlFromNavigationEndpoint, and so by getTextFromObject, getUrlFromObject and getTextAtKey, the methods which were catching ParsingExceptions thrown by these methods had to be updated. URLs got in the HTML version of getTextFromObject are now escaped properly to provide valid HTML to clients. This has been also done for attribute descriptions, with the description text for this type of descriptions. As YouTube descriptions are in HTML format (except for the fallback on the JSON player response, which is plain text and only happens when there is no visual metadata or a breaking change), all URLs returned are escaped, so tests which are testing presence of URLs with escaped characters had to be updated (it was only the case for YoutubeStreamExtractorDefaultTest.DescriptionTestUnboxing).
This commit is contained in:
parent
99ab9777ad
commit
1556adbb2d
|
@ -822,7 +822,7 @@ public final class YoutubeParsingHelper {
|
|||
|
||||
try {
|
||||
final String url = "https://music.youtube.com/sw.js";
|
||||
final var headers = getOriginReferrerHeaders("https://music.youtube.com");
|
||||
final var headers = getOriginReferrerHeaders(YOUTUBE_MUSIC_URL);
|
||||
final String response = getDownloader().get(url, headers).responseBody();
|
||||
musicClientVersion = getStringResultFromRegexArray(response,
|
||||
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
|
||||
|
@ -843,18 +843,11 @@ public final class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
@Nullable
|
||||
public static String getUrlFromNavigationEndpoint(@Nonnull final JsonObject navigationEndpoint)
|
||||
throws ParsingException {
|
||||
if (navigationEndpoint.has("webCommandMetadata")) {
|
||||
// this case needs to be handled before the browseEndpoint,
|
||||
// e.g. for hashtags in comments
|
||||
final JsonObject metadata = navigationEndpoint.getObject("webCommandMetadata");
|
||||
if (metadata.has("url")) {
|
||||
return "https://www.youtube.com" + metadata.getString("url");
|
||||
}
|
||||
}
|
||||
public static String getUrlFromNavigationEndpoint(
|
||||
@Nonnull final JsonObject navigationEndpoint) {
|
||||
if (navigationEndpoint.has("urlEndpoint")) {
|
||||
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
|
||||
String internUrl = navigationEndpoint.getObject("urlEndpoint")
|
||||
.getString("url");
|
||||
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
|
||||
// remove https://www.youtube.com part to fall in the next if block
|
||||
internUrl = internUrl.substring(23);
|
||||
|
@ -879,7 +872,9 @@ public final class YoutubeParsingHelper {
|
|||
|| internUrl.startsWith("/watch")) {
|
||||
return "https://www.youtube.com" + internUrl;
|
||||
}
|
||||
} else if (navigationEndpoint.has("browseEndpoint")) {
|
||||
}
|
||||
|
||||
if (navigationEndpoint.has("browseEndpoint")) {
|
||||
final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
|
||||
final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
|
||||
final String browseId = browseEndpoint.getString("browseId");
|
||||
|
@ -892,26 +887,39 @@ public final class YoutubeParsingHelper {
|
|||
if (!isNullOrEmpty(canonicalBaseUrl)) {
|
||||
return "https://www.youtube.com" + canonicalBaseUrl;
|
||||
}
|
||||
}
|
||||
|
||||
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\""
|
||||
+ browseEndpoint + "\")");
|
||||
} else if (navigationEndpoint.has("watchEndpoint")) {
|
||||
if (navigationEndpoint.has("watchEndpoint")) {
|
||||
final StringBuilder url = new StringBuilder();
|
||||
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
|
||||
.getObject("watchEndpoint").getString(VIDEO_ID));
|
||||
url.append("https://www.youtube.com/watch?v=")
|
||||
.append(navigationEndpoint.getObject("watchEndpoint")
|
||||
.getString(VIDEO_ID));
|
||||
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
|
||||
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
|
||||
.getString("playlistId"));
|
||||
}
|
||||
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
|
||||
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint")
|
||||
url.append("&t=")
|
||||
.append(navigationEndpoint.getObject("watchEndpoint")
|
||||
.getInt("startTimeSeconds"));
|
||||
}
|
||||
return url.toString();
|
||||
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
|
||||
return "https://www.youtube.com/playlist?list="
|
||||
+ navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
|
||||
}
|
||||
|
||||
if (navigationEndpoint.has("watchPlaylistEndpoint")) {
|
||||
return "https://www.youtube.com/playlist?list="
|
||||
+ navigationEndpoint.getObject("watchPlaylistEndpoint")
|
||||
.getString("playlistId");
|
||||
}
|
||||
|
||||
if (navigationEndpoint.has("commandMetadata")) {
|
||||
final JsonObject metadata = navigationEndpoint.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata");
|
||||
if (metadata.has("url")) {
|
||||
return "https://www.youtube.com" + metadata.getString("url");
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -924,8 +932,7 @@ public final class YoutubeParsingHelper {
|
|||
* @return text in the JSON object or {@code null}
|
||||
*/
|
||||
@Nullable
|
||||
public static String getTextFromObject(final JsonObject textObject, final boolean html)
|
||||
throws ParsingException {
|
||||
public static String getTextFromObject(final JsonObject textObject, final boolean html) {
|
||||
if (isNullOrEmpty(textObject)) {
|
||||
return null;
|
||||
}
|
||||
|
@ -944,12 +951,12 @@ public final class YoutubeParsingHelper {
|
|||
String text = run.getString("text");
|
||||
|
||||
if (html) {
|
||||
text = Entities.escape(text);
|
||||
if (run.has("navigationEndpoint")) {
|
||||
final String url = getUrlFromNavigationEndpoint(run
|
||||
.getObject("navigationEndpoint"));
|
||||
final String url = getUrlFromNavigationEndpoint(
|
||||
run.getObject("navigationEndpoint"));
|
||||
if (!isNullOrEmpty(url)) {
|
||||
text = "<a href=\"" + url + "\">" + text + "</a>";
|
||||
text = "<a href=\"" + Entities.escape(url) + "\">" + Entities.escape(text)
|
||||
+ "</a>";
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1015,11 +1022,12 @@ public final class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
final String content = attributedDescription.getString("content");
|
||||
final JsonArray commandRuns = attributedDescription.getArray("commandRuns");
|
||||
if (content == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final JsonArray commandRuns = attributedDescription.getArray("commandRuns");
|
||||
|
||||
final StringBuilder textBuilder = new StringBuilder();
|
||||
int textStart = 0;
|
||||
|
||||
|
@ -1038,12 +1046,7 @@ public final class YoutubeParsingHelper {
|
|||
continue;
|
||||
}
|
||||
|
||||
final String url;
|
||||
try {
|
||||
url = getUrlFromNavigationEndpoint(navigationEndpoint);
|
||||
} catch (final ParsingException e) {
|
||||
continue;
|
||||
}
|
||||
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
|
||||
|
||||
if (url == null) {
|
||||
continue;
|
||||
|
@ -1062,9 +1065,9 @@ public final class YoutubeParsingHelper {
|
|||
.replaceFirst("^[/•] *", "");
|
||||
|
||||
textBuilder.append("<a href=\"")
|
||||
.append(url)
|
||||
.append(Entities.escape(url))
|
||||
.append("\">")
|
||||
.append(linkText)
|
||||
.append(Entities.escape(linkText))
|
||||
.append("</a>");
|
||||
|
||||
textStart = startIndex + length;
|
||||
|
@ -1081,13 +1084,12 @@ public final class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
@Nullable
|
||||
public static String getTextFromObject(final JsonObject textObject) throws ParsingException {
|
||||
public static String getTextFromObject(final JsonObject textObject) {
|
||||
return getTextFromObject(textObject, false);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static String getUrlFromObject(final JsonObject textObject) throws ParsingException {
|
||||
|
||||
public static String getUrlFromObject(final JsonObject textObject) {
|
||||
if (isNullOrEmpty(textObject)) {
|
||||
return null;
|
||||
}
|
||||
|
@ -1108,8 +1110,7 @@ public final class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
@Nullable
|
||||
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey)
|
||||
throws ParsingException {
|
||||
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey) {
|
||||
if (jsonObject.isString(theKey)) {
|
||||
return jsonObject.getString(theKey);
|
||||
} else {
|
||||
|
|
|
@ -45,14 +45,11 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
|||
this.channelInfoItem = channelInfoItem;
|
||||
|
||||
boolean wHandle = false;
|
||||
try {
|
||||
final String subscriberCountText = getTextFromObject(
|
||||
channelInfoItem.getObject("subscriberCountText"));
|
||||
if (subscriberCountText != null) {
|
||||
wHandle = subscriberCountText.startsWith("@");
|
||||
}
|
||||
} catch (final ParsingException ignored) {
|
||||
}
|
||||
this.withHandle = wHandle;
|
||||
}
|
||||
|
||||
|
|
|
@ -168,11 +168,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
title = playerResponse.getObject("videoDetails").getString("title");
|
||||
|
||||
if (isNullOrEmpty(title)) {
|
||||
try {
|
||||
title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
|
||||
} catch (final ParsingException ignored) {
|
||||
// Age-restricted videos cause a ParsingException here
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(title)) {
|
||||
throw new ParsingException("Could not get name");
|
||||
|
@ -285,12 +281,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public Description getDescription() throws ParsingException {
|
||||
assertPageFetched();
|
||||
// Description with more info on links
|
||||
try {
|
||||
final String description = getTextFromObject(
|
||||
final String videoSecondaryInfoRendererDescription = getTextFromObject(
|
||||
getVideoSecondaryInfoRenderer().getObject("description"),
|
||||
true);
|
||||
if (!isNullOrEmpty(description)) {
|
||||
return new Description(description, Description.HTML);
|
||||
if (!isNullOrEmpty(videoSecondaryInfoRendererDescription)) {
|
||||
return new Description(videoSecondaryInfoRendererDescription, Description.HTML);
|
||||
}
|
||||
|
||||
final String attributedDescription = getAttributedDescription(
|
||||
|
@ -298,9 +293,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
if (!isNullOrEmpty(attributedDescription)) {
|
||||
return new Description(attributedDescription, Description.HTML);
|
||||
}
|
||||
} catch (final ParsingException ignored) {
|
||||
// Age-restricted videos cause a ParsingException here
|
||||
}
|
||||
|
||||
String description = playerResponse.getObject("videoDetails")
|
||||
.getString("shortDescription");
|
||||
|
@ -400,14 +392,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
String views = null;
|
||||
|
||||
try {
|
||||
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
|
||||
String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
|
||||
.getObject("videoViewCountRenderer").getObject("viewCount"));
|
||||
} catch (final ParsingException ignored) {
|
||||
// Age-restricted videos cause a ParsingException here
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(views)) {
|
||||
views = playerResponse.getObject("videoDetails").getString("viewCount");
|
||||
|
@ -795,7 +781,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return getTextFromObject(playerResponse.getObject("playabilityStatus")
|
||||
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
|
||||
.getObject("reason"));
|
||||
} catch (final ParsingException | NullPointerException e) {
|
||||
} catch (final NullPointerException e) {
|
||||
return null; // No error message
|
||||
}
|
||||
}
|
||||
|
|
|
@ -183,10 +183,10 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34");
|
||||
return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34");
|
||||
}
|
||||
@Override public long expectedLength() { return 434; }
|
||||
@Override public long expectedViewCountAtLeast() { return 21229200; }
|
||||
|
|
Loading…
Reference in New Issue