be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #604 from TiA4f8R/youtubei-api 

[YouTube] Use the new internal API in NewPipe Extractor
This commit is contained in:
Stypox 2021-08-03 19:01:14 +02:00 committed by GitHub
parent c97a19d719 79cc5c8d12
commit 5a88263785
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
254 changed files with 48467 additions and 15853 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -17,7 +18,7 @@ public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem>
* @apiNote Warning: This method is experimental and may get removed in a future release.
* @return <code>true</code> if the comments are disabled otherwise <code>false</code> (default)
*/
public boolean isCommentsDisabled() {
public boolean isCommentsDisabled() throws ExtractionException {
return false;
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -46,13 +46,12 @@ public class ItagItem {
/// VIDEO ONLY ////////////////////////////////////////////
// ID Type Format Resolution FPS ///
/////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
// new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
// new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
@ -75,6 +74,7 @@ public class ItagItem {
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
};
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java
View File

@ -60,6 +60,14 @@ public class YoutubeJavaScriptExtractor {
return extractJavaScriptCode("d4IGg5dqeO8");
}
/**
* Reset the JavaScript code. It will be fetched again the next time
* {@link #extractJavaScriptCode()} or {@link #extractJavaScriptCode(String)} is called.
*/
public static void resetJavaScriptCode() {
cachedJavaScriptCode = null;
}
private static String extractJavaScriptUrl(final String videoId) throws ParsingException {
try {
final String embedUrl = "https://www.youtube.com/embed/" + videoId;

492
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -37,7 +38,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -64,13 +64,22 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
private static final String HARDCODED_CLIENT_VERSION = "2.20210728.00.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.29.38";
private static String clientVersion;
private static String key;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
private static String[] youtubeMusicKeys;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210726.00.01"};
private static String[] youtubeMusicKey;
private static boolean keyAndVersionExtracted = false;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
private static Random numberGenerator = new Random();
@ -85,7 +94,8 @@ public class YoutubeParsingHelper {
*/
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_CHANNEL_ID =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static boolean isGoogleURL(String url) {
@ -93,30 +103,34 @@ public class YoutubeParsingHelper {
try {
final URL u = new URL(url);
final String host = u.getHost();
return host.startsWith("google.") || host.startsWith("m.google.")
return host.startsWith("google.")
|| host.startsWith("m.google.")
|| host.startsWith("www.google.");
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
return false;
}
}
public static boolean isYoutubeURL(final URL url) {
public static boolean isYoutubeURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
return host.equalsIgnoreCase("youtube.com")
|| host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com")
|| host.equalsIgnoreCase("music.youtube.com");
}
public static boolean isYoutubeServiceURL(final URL url) {
public static boolean isYoutubeServiceURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
return host.equalsIgnoreCase("www.youtube-nocookie.com")
|| host.equalsIgnoreCase("youtu.be");
}
public static boolean isHooktubeURL(final URL url) {
public static boolean isHooktubeURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("hooktube.com");
}
public static boolean isInvidioURL(final URL url) {
public static boolean isInvidioURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("invidio.us")
|| host.equalsIgnoreCase("dev.invidio.us")
@ -153,7 +167,7 @@ public class YoutubeParsingHelper {
* @return the duration in seconds
* @throws ParsingException when more than 3 separators are found
*/
public static int parseDurationString(final String input)
public static int parseDurationString(@Nonnull final String input)
throws ParsingException, NumberFormatException {
// If time separator : is not detected, try . instead
final String[] splitInput = input.contains(":")
@ -194,7 +208,8 @@ public class YoutubeParsingHelper {
+ Integer.parseInt(Utils.removeNonDigitCharacters(seconds));
}
public static String getFeedUrlFrom(final String channelIdOrUser) {
@Nonnull
public static String getFeedUrlFrom(@Nonnull final String channelIdOrUser) {
if (channelIdOrUser.startsWith("user/")) {
return FEED_BASE_USER + channelIdOrUser.replace("user/", "");
} else if (channelIdOrUser.startsWith("channel/")) {
@ -204,14 +219,16 @@ public class YoutubeParsingHelper {
}
}
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
public static OffsetDateTime parseDateFrom(final String textualUploadDate)
throws ParsingException {
try {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
} catch (final DateTimeParseException e) {
try {
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
} catch (DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
} catch (final DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"",
e1);
}
}
}
@ -220,10 +237,10 @@ public class YoutubeParsingHelper {
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
*
* @param playlistId
* @param playlistId the playlist id
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(final String playlistId) {
public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
}
@ -231,10 +248,10 @@ public class YoutubeParsingHelper {
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
*
* @param playlistId
* @param playlistId the playlist id
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
public static boolean isYoutubeMusicMixId(@Nonnull final String playlistId) {
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
@ -244,7 +261,7 @@ public class YoutubeParsingHelper {
*
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
public static boolean isYoutubeChannelMixId(@Nonnull final String playlistId) {
return playlistId.startsWith("RDCM");
}
@ -253,7 +270,9 @@ public class YoutubeParsingHelper {
*
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
@Nonnull
public static String extractVideoIdFromMixId(@Nonnull final String playlistId)
throws ParsingException {
if (playlistId.startsWith("RDMM")) { // My Mix
return playlistId.substring(4);
@ -262,49 +281,88 @@ public class YoutubeParsingHelper {
} else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
// Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
throw new ParsingException("Video id could not be determined from mix id: "
+ playlistId);
} else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
return playlistId.substring(2);
} else { // not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
throw new ParsingException("Video id could not be determined from mix id: "
+ playlistId);
}
}
public static JsonObject getInitialData(final String html) throws ParsingException {
try {
try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
final String initialData = Parser.matchGroup1(
"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
} catch (Parser.RegexException e) {
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
} catch (final Parser.RegexException e) {
final String initialData = Parser.matchGroup1(
"var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
}
} catch (JsonParserException | Parser.RegexException e) {
} catch (final JsonParserException | Parser.RegexException e) {
throw new ParsingException("Could not get ytInitialData", e);
}
}
public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException {
final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
public static boolean areHardcodedClientVersionAndKeyValid()
throws IOException, ExtractionException {
if (hardcodedClientVersionAndKeyValid.isPresent()) {
return hardcodedClientVersionAndKeyValid.get();
}
// @formatter:off
final byte[] body = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("hl", "en-GB")
.value("gl", "GB")
.value("clientName", "WEB")
.value("clientVersion", HARDCODED_CLIENT_VERSION)
.end()
.object("user")
.value("lockedSafetyMode", false)
.end()
.value("fetchLiveState", true)
.end()
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers).responseBody();
headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION));
return response.length() > 50; // ensure to have a valid response
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
+ HARDCODED_KEY, headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
&& responseCode == 200); // Ensure to have a valid response
return hardcodedClientVersionAndKeyValid.get();
}
private static void extractClientVersionAndKey() throws IOException, ExtractionException {
final String url = "https://www.youtube.com/results?search_query=test";
final String html = getDownloader().get(url).responseBody();
// Don't extract the client version and the InnerTube key if it has been already extracted
if (keyAndVersionExtracted) return;
// Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
final Map<String, List<String>> headers = new HashMap<>();
addCookieHeader(headers);
final String html = getDownloader().get(url, headers).responseBody();
final JsonObject initialData = getInitialData(html);
final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
final JsonArray serviceTrackingParams = initialData.getObject("responseContext")
.getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first
// Try to get version from initial data first
for (final Object service : serviceTrackingParams) {
final JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
@ -317,7 +375,8 @@ public class YoutubeParsingHelper {
}
}
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last two digits
// Fallback to get a shortened client version which does not contain the last two
// digits
final JsonArray params = s.getArray("params");
for (final Object param : params) {
final JsonObject p = (JsonObject) param;
@ -342,7 +401,7 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion;
break;
}
} catch (Parser.RegexException ignored) {
} catch (final Parser.RegexException ignored) {
}
}
@ -352,12 +411,14 @@ public class YoutubeParsingHelper {
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e1) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) {
} catch (final Parser.RegexException e2) {
throw new ParsingException("Could not extract client version and key");
}
}
keyAndVersionExtracted = true;
}
/**
@ -365,10 +426,11 @@ public class YoutubeParsingHelper {
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
if (areHardcodedClientVersionAndKeyValid()) {
return clientVersion = HARDCODED_CLIENT_VERSION;
}
extractClientVersionAndKey();
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
return clientVersion;
}
@ -377,9 +439,11 @@ public class YoutubeParsingHelper {
*/
public static String getKey() throws IOException, ExtractionException {
if (!isNullOrEmpty(key)) return key;
if (areHardcodedClientVersionAndKeyValid()) {
return key = HARDCODED_KEY;
}
extractClientVersionAndKey();
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
return key;
}
@ -408,12 +472,15 @@ public class YoutubeParsingHelper {
* <b>Only use in tests.</b>
* </p>
*/
public static void setNumberGenerator(Random random) {
public static void setNumberGenerator(final Random random) {
numberGenerator = random;
}
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
ReCaptchaException {
final String url =
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
+ HARDCODED_YOUTUBE_MUSIC_KEY[0];
// @formatter:off
byte[] json = JsonWriter.string()
@ -421,12 +488,11 @@ public class YoutubeParsingHelper {
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2])
.value("hl", "en")
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2])
.value("hl", "en-GB")
.value("gl", "GB")
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.value("experimentsToken", EMPTY_STRING)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
@ -440,58 +506,66 @@ public class YoutubeParsingHelper {
.value("enableSafetyMode", false)
.end()
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.value("input", "")
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2]));
headers.put("X-YouTube-Client-Name", Collections.singletonList(
HARDCODED_YOUTUBE_MUSIC_KEY[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
addCookieHeader(headers);
final String response = getDownloader().post(url, headers, json).responseBody();
return response.length() > 50; // ensure to have a valid response
final Response response = getDownloader().post(url, headers, json);
// Ensure to have a valid response
return response.responseBody().length() > 500 && response.responseCode() == 200;
}
public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException {
if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys;
if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS;
public static String[] getYoutubeMusicKey() throws IOException, ReCaptchaException,
Parser.RegexException {
if (youtubeMusicKey != null && youtubeMusicKey.length == 3) return youtubeMusicKey;
if (isHardcodedYoutubeMusicKeyValid()) {
return youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY;
}
final String url = "https://music.youtube.com/";
final String html = getDownloader().get(url).responseBody();
final Map<String, List<String>> headers = new HashMap<>();
addCookieHeader(headers);
final String html = getDownloader().get(url, headers).responseBody();
String key;
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
}
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),",
html);
String clientVersion;
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException e) {
clientVersion = Parser.matchGroup1(
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException e) {
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException ee) {
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
clientVersion = Parser.matchGroup1(
"INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1(
"innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
}
}
return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
return youtubeMusicKey = new String[]{key, clientName, clientVersion};
}
@Nullable
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
public static String getUrlFromNavigationEndpoint(@Nonnull final JsonObject navigationEndpoint)
throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
@ -508,7 +582,7 @@ public class YoutubeParsingHelper {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
return null;
}
return url;
@ -516,7 +590,8 @@ public class YoutubeParsingHelper {
}
} else if (internUrl.startsWith("http")) {
return internUrl;
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) {
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user")
|| internUrl.startsWith("/watch")) {
return "https://www.youtube.com" + internUrl;
}
} else if (navigationEndpoint.has("browseEndpoint")) {
@ -533,10 +608,12 @@ public class YoutubeParsingHelper {
return "https://www.youtube.com" + canonicalBaseUrl;
}
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")");
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\""
+ browseEndpoint + "\")");
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId"));
@ -561,7 +638,8 @@ public class YoutubeParsingHelper {
* @return text in the JSON object or {@code null}
*/
@Nullable
public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException {
public static String getTextFromObject(final JsonObject textObject, final boolean html)
throws ParsingException {
if (isNullOrEmpty(textObject)) return null;
if (textObject.has("simpleText")) return textObject.getString("simpleText");
@ -572,9 +650,11 @@ public class YoutubeParsingHelper {
for (final Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
textBuilder.append("<a href=\"").append(url).append("\">").append(text)
.append("</a>");
continue;
}
}
@ -592,12 +672,12 @@ public class YoutubeParsingHelper {
}
@Nullable
public static String getTextFromObject(JsonObject textObject) throws ParsingException {
public static String getTextFromObject(final JsonObject textObject) throws ParsingException {
return getTextFromObject(textObject, false);
}
@Nullable
public static String getTextAtKey(final JsonObject jsonObject, final String key)
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String key)
throws ParsingException {
if (jsonObject.isString(key)) {
return jsonObject.getString(key);
@ -606,7 +686,7 @@ public class YoutubeParsingHelper {
}
}
public static String fixThumbnailUrl(String thumbnailUrl) {
public static String fixThumbnailUrl(@Nonnull String thumbnailUrl) {
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = thumbnailUrl.substring(2);
}
@ -620,7 +700,8 @@ public class YoutubeParsingHelper {
return thumbnailUrl;
}
public static String getValidJsonResponseBody(final Response response)
@Nonnull
public static String getValidJsonResponseBody(@Nonnull final Response response)
throws ParsingException, MalformedURLException {
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found"
@ -628,7 +709,7 @@ public class YoutubeParsingHelper {
}
final String responseBody = response.responseBody();
if (responseBody.length() < 50) { // ensure to have a valid response
if (responseBody.length() < 50) { // Ensure to have a valid response
throw new ParsingException("JSON response is too short");
}
@ -662,6 +743,41 @@ public class YoutubeParsingHelper {
return response;
}
public static JsonObject getJsonPostResponse(final String endpoint,
final byte[] body,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
+ getKey(), headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonObject getJsonMobilePostResponse(final String endpoint,
final byte[] body,
@Nonnull final ContentCountry
contentCountry,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Content-Type", Collections.singletonList("application/json"));
// Spoofing an Android 11 device with the hardcoded version of the Android app
headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/"
+ MOBILE_YOUTUBE_CLIENT_VERSION + "Linux; U; Android 11; "
+ contentCountry.getCountryCode() + ") gzip"));
headers.put("x-goog-api-format-version", Collections.singletonList("2"));
final Response response = getDownloader().post(
"https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key="
+ MOBILE_YOUTUBE_KEY, headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
@ -672,7 +788,8 @@ public class YoutubeParsingHelper {
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
public static JsonArray getJsonResponse(@Nonnull final Page page,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
@ -682,19 +799,140 @@ public class YoutubeParsingHelper {
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonBuilder<JsonObject> prepareJsonBuilder()
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "1")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("clientScreen", "EMBED")
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
.value("clientScreen", "EMBED")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
// @formatter:on
}
@Nonnull
public static byte[] createPlayerBodyWithSts(final Localization localization,
final ContentCountry contentCountry,
final String videoId,
final boolean withThirdParty,
@Nullable final String sts)
throws IOException, ExtractionException {
if (withThirdParty) {
// @formatter:off
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
} else {
// @formatter:off
return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
.value("videoId", videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
}
}
/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)
@ -707,14 +945,17 @@ public class YoutubeParsingHelper {
}
/**
* Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
* Add the <code>X-YouTube-Client-Name</code>, <code>X-YouTube-Client-Version</code>,
* <code>Origin</code>, and <code>Referer</code> headers.
* @param headers The headers which should be completed
*/
public static void addClientInfoHeaders(final Map<String, List<String>> headers)
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
throws IOException, ExtractionException {
if (headers.get("X-YouTube-Client-Name") == null) {
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
}
headers.computeIfAbsent("Origin", k -> Collections.singletonList(
"https://www.youtube.com"));
headers.computeIfAbsent("Referer", k -> Collections.singletonList(
"https://www.youtube.com"));
headers.computeIfAbsent("X-YouTube-Client-Name", k -> Collections.singletonList("1"));
if (headers.get("X-YouTube-Client-Version") == null) {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
}
@ -725,7 +966,7 @@ public class YoutubeParsingHelper {
* @see #CONSENT_COOKIE
* @param headers the headers which should be completed
*/
public static void addCookieHeader(final Map<String, List<String>> headers) {
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
} else {
@ -733,12 +974,14 @@ public class YoutubeParsingHelper {
}
}
@Nonnull
public static String generateConsentCookie() {
final int statusCode = 100 + numberGenerator.nextInt(900);
return CONSENT_COOKIE + statusCode;
}
public static String extractCookieValue(final String cookieName, final Response response) {
public static String extractCookieValue(final String cookieName,
@Nonnull final Response response) {
final List<String> cookies = response.responseHeaders().get("set-cookie");
int startIndex;
String result = "";
@ -761,7 +1004,8 @@ public class YoutubeParsingHelper {
* @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected
*/
public static void defaultAlertsCheck(final JsonObject initialData) throws ParsingException {
public static void defaultAlertsCheck(@Nonnull final JsonObject initialData)
throws ParsingException {
final JsonArray alerts = initialData.getArray("alerts");
if (!isNullOrEmpty(alerts)) {
final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
@ -771,7 +1015,7 @@ public class YoutubeParsingHelper {
if (alertText != null && alertText.contains("This account has been terminated")) {
if (alertText.contains("violation") || alertText.contains("violating")
|| alertText.contains("infringement")) {
// possible error messages:
// Possible error messages:
// "This account has been terminated for a violation of YouTube's Terms of Service."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten."
@ -791,7 +1035,8 @@ public class YoutubeParsingHelper {
}
@Nonnull
public static List<MetaInfo> getMetaInfo(final JsonArray contents) throws ParsingException {
public static List<MetaInfo> getMetaInfo(@Nonnull final JsonArray contents)
throws ParsingException {
final List<MetaInfo> metaInfo = new ArrayList<>();
for (final Object content : contents) {
final JsonObject resultObject = (JsonObject) content;
@ -801,10 +1046,12 @@ public class YoutubeParsingHelper {
final JsonObject sectionContent = (JsonObject) sectionContentObject;
if (sectionContent.has("infoPanelContentRenderer")) {
metaInfo.add(getInfoPanelContent(sectionContent.getObject("infoPanelContentRenderer")));
metaInfo.add(getInfoPanelContent(sectionContent
.getObject("infoPanelContentRenderer")));
}
if (sectionContent.has("clarificationRenderer")) {
metaInfo.add(getClarificationRendererContent(sectionContent.getObject("clarificationRenderer")
metaInfo.add(getClarificationRendererContent(sectionContent
.getObject("clarificationRenderer")
));
}
@ -815,7 +1062,7 @@ public class YoutubeParsingHelper {
}
@Nonnull
private static MetaInfo getInfoPanelContent(final JsonObject infoPanelContentRenderer)
private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer)
throws ParsingException {
final MetaInfo metaInfo = new MetaInfo();
final StringBuilder sb = new StringBuilder();
@ -830,7 +1077,8 @@ public class YoutubeParsingHelper {
final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint(
infoPanelContentRenderer.getObject("sourceEndpoint"));
try {
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(metaInfoLinkUrl))));
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(
metaInfoLinkUrl))));
} catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e);
}
@ -847,12 +1095,14 @@ public class YoutubeParsingHelper {
}
@Nonnull
private static MetaInfo getClarificationRendererContent(final JsonObject clarificationRenderer)
private static MetaInfo getClarificationRendererContent(@Nonnull final JsonObject clarificationRenderer)
throws ParsingException {
final MetaInfo metaInfo = new MetaInfo();
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
.getObject("text"));
if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content");
}
@ -863,7 +1113,8 @@ public class YoutubeParsingHelper {
final JsonObject actionButton = clarificationRenderer.getObject("actionButton")
.getObject("buttonRenderer");
try {
final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton.getObject("command"));
final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton
.getObject("command"));
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
} catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e);
@ -877,15 +1128,18 @@ public class YoutubeParsingHelper {
metaInfo.addUrlText(metaInfoLinkText);
}
if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer.has("secondarySource")) {
final String url = getUrlFromNavigationEndpoint(clarificationRenderer.getObject("secondaryEndpoint"));
// ignore Google URLs, because those point to a Google search about "Covid-19"
if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer
.has("secondarySource")) {
final String url = getUrlFromNavigationEndpoint(clarificationRenderer
.getObject("secondaryEndpoint"));
// Ignore Google URLs, because those point to a Google search about "Covid-19"
if (url != null && !isGoogleURL(url)) {
try {
metaInfo.addUrl(new URL(url));
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource"));
final String description = getTextFromObject(clarificationRenderer
.getObject("secondarySource"));
metaInfo.addUrlText(description == null ? url : description);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new ParsingException("Could not get metadata info secondary URL", e);
}
}
@ -928,7 +1182,8 @@ public class YoutubeParsingHelper {
return false;
}
public static String unescapeDocument(final String doc) {
@Nonnull
public static String unescapeDocument(@Nonnull final String doc) {
return doc
.replaceAll("\\\\x22", "\"")
.replaceAll("\\\\x7b", "{")
@ -936,5 +1191,4 @@ public class YoutubeParsingHelper {
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View File

@ -79,11 +79,13 @@ public class YoutubeThrottlingDecrypter {
}
}
@Nonnull
private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
final String functionBase = functionName + "=function";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
}
@Nonnull
private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
Pattern.DOTALL);

233
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -22,15 +23,15 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -72,35 +73,110 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/
private String redirectedChannelId;
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
JsonArray ajaxJson = null;
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String channelPath = super.getId();
final String[] channelId = channelPath.split("/");
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(UTF_8);
int level = 0;
while (level < 3) {
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
final JsonObject endpoint = jsonResponse.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction")
.getObject("endpoint");
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata")
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", EMPTY_STRING);
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", EMPTY_STRING);
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") && !browseId.isEmpty()) {
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid";
id = browseId;
redirectedChannelId = browseId;
}
} else {
id = channelId[1];
}
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
.getBytes(UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", EMPTY_STRING);
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
EMPTY_STRING);
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
level++;
} else {
@ -113,7 +189,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ExtractionException("Could not fetch initial JSON data");
}
initialData = ajaxJson.getObject(1).getObject("response");
initialData = ajaxJson;
YoutubeParsingHelper.defaultAlertsCheck(initialData);
}
@ -122,7 +198,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getUrl() throws ParsingException {
try {
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
} catch (ParsingException e) {
} catch (final ParsingException e) {
return super.getUrl();
}
}
@ -130,7 +206,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
final String channelId = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", EMPTY_STRING);
if (!channelId.isEmpty()) {
@ -146,8 +223,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getName() throws ParsingException {
try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
} catch (Exception e) {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("title");
} catch (final Exception e) {
throw new ParsingException("Could not get channel name", e);
}
}
@ -155,11 +233,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
.getArray("thumbnails").getObject(0).getString("url");
String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get avatar", e);
}
}
@ -167,15 +246,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
.getArray("thumbnails").getObject(0).getString("url");
String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails")
.getObject(0).getString("url");
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null;
}
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get banner", e);
}
}
@ -184,18 +264,20 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getFeedUrl() throws ParsingException {
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@Override
public long getSubscriberCount() throws ParsingException {
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer");
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer");
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
try {
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText")));
} catch (NumberFormatException e) {
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
.getObject("subscriberCountText")));
} catch (final NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
@ -206,30 +288,32 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getDescription() throws ParsingException {
try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
} catch (Exception e) {
return initialData.getObject("metadata").getObject("channelMetadataRenderer")
.getString("description");
} catch (final Exception e) {
throw new ParsingException("Could not get channel description", e);
}
}
@Override
public String getParentChannelName() throws ParsingException {
public String getParentChannelName() {
return "";
}
@Override
public String getParentChannelUrl() throws ParsingException {
public String getParentChannelUrl() {
return "";
}
@Override
public String getParentChannelAvatarUrl() throws ParsingException {
public String getParentChannelAvatarUrl() {
return "";
}
@Override
public boolean isVerified() throws ParsingException {
final JsonArray badges = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
final JsonArray badges = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getArray("badges");
return YoutubeParsingHelper.isVerified(badges);
@ -243,29 +327,36 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
Page nextPage = null;
if (getVideoTab() != null) {
final JsonObject gridRenderer = getVideoTab().getObject("content").getObject("sectionListRenderer")
final JsonObject gridRenderer = getVideoTab().getObject("content")
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer");
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer.getArray("items"));
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
.getArray("items"), channelIds);
nextPage = getNextPageFrom(continuation);
nextPage = getNextPageFrom(continuation, channelIds);
}
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
// Unfortunately, we have to fetch the page even if we are only getting next streams,
// as they don't deliver enough information on their own (the channel name, for example).
fetchPage();
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
getExtractorLocalization());
@ -275,46 +366,53 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
body);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be commited
* @param videos the array to get videos from
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
* @throws ParsingException if an error happened while extracting
*/
private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = getName();
final String uploaderUrl = getUrl();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (Object object : videos) {
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
@ -337,16 +435,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab;
JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject videoTab = null;
for (Object tab : tabs) {
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}

249
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -1,8 +1,18 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@ -10,38 +20,19 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
public class YoutubeCommentsExtractor extends CommentsExtractor {
// using the mobile site for comments because it loads faster and uses get requests instead of post
private static final String USER_AGENT = "Mozilla/5.0 (Android 9; Mobile; rv:78.0) Gecko/20100101 Firefox/78.0";
private static final Pattern YT_CLIENT_NAME_PATTERN = Pattern.compile("INNERTUBE_CONTEXT_CLIENT_NAME\\\":(.*?)[,}]");
private String ytClientVersion;
private String ytClientName;
private String responseBody;
private JsonObject nextResponse;
/**
* Caching mechanism and holder of the commentsDisabled value.
@ -52,6 +43,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
* If the method or another one that is depending on disabled comments
* is now called again, the method execution can avoid unnecessary calls
*/
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<Boolean> optCommentsDisabled = Optional.empty();
public YoutubeCommentsExtractor(
@ -60,6 +52,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
super(service, uiHandler);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage()
throws IOException, ExtractionException {
@ -81,163 +74,177 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
/**
* Finds the initial comments token and initializes commentsDisabled.
*
* @return the continuation token or null if none was found
*/
private String findInitialCommentsToken() {
final String continuationStartPattern = "continuation\":\"";
@Nullable
private String findInitialCommentsToken() throws ExtractionException {
String commentsTokenInside = findValue(responseBody, "sectionListRenderer", "}");
if (commentsTokenInside == null || !commentsTokenInside.contains(continuationStartPattern)) {
commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
final JsonArray jArray = JsonUtils.getArray(nextResponse,
"contents.twoColumnWatchNextResults.results.results.contents");
final Optional<Object> itemSectionRenderer = jArray.stream().filter(o -> {
JsonObject jObj = (JsonObject) o;
if (jObj.has("itemSectionRenderer")) {
try {
return JsonUtils.getString(jObj, "itemSectionRenderer.targetId")
.equals("comments-section");
} catch (final ParsingException ignored) {
}
}
return false;
}).findFirst();
final String token;
if (itemSectionRenderer.isPresent()) {
token = JsonUtils.getString(((JsonObject) itemSectionRenderer.get())
.getObject("itemSectionRenderer").getArray("contents").getObject(0),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
} else {
token = null;
}
// If no continuation token is found the comments are disabled
if (commentsTokenInside == null || !commentsTokenInside.contains(continuationStartPattern)) {
if (token == null) {
optCommentsDisabled = Optional.of(true);
return null;
}
// If a continuation token is found there are >= 0 comments
final String commentsToken = findValue(commentsTokenInside, continuationStartPattern, "\"");
optCommentsDisabled = Optional.of(false);
return commentsToken;
return token;
}
@Nonnull
private InfoItemsPage<CommentsInfoItem> getInfoItemsPageForDisabledComments() {
return new InfoItemsPage<>(Collections.emptyList(), null, Collections.emptyList());
}
private Page getNextPage(final JsonObject ajaxJson) throws ParsingException {
final JsonArray arr;
@Nullable
private Page getNextPage(@Nonnull final JsonObject ajaxJson) throws ExtractionException {
final JsonArray jsonArray;
final JsonArray onResponseReceivedEndpoints = ajaxJson.getArray(
"onResponseReceivedEndpoints");
final JsonObject endpoint = onResponseReceivedEndpoints.getObject(
onResponseReceivedEndpoints.size() - 1);
try {
arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
jsonArray = endpoint.getObject("reloadContinuationItemsCommand", endpoint.getObject(
"appendContinuationItemsAction")).getArray("continuationItems");
} catch (final Exception e) {
return null;
}
if (arr.isEmpty()) {
if (jsonArray.isEmpty()) {
return null;
}
final String continuation;
try {
continuation = JsonUtils.getString(arr.getObject(0), "nextContinuationData.continuation");
continuation = JsonUtils.getString(jsonArray.getObject(jsonArray.size() - 1),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
} catch (final Exception e) {
return null;
}
return getNextPage(continuation);
}
@Nonnull
private Page getNextPage(final String continuation) throws ParsingException {
final Map<String, String> params = new HashMap<>();
params.put("action_get_comments", "1");
params.put("pbj", "1");
params.put("ctoken", continuation);
try {
return new Page("https://m.youtube.com/watch_comment?" + getDataString(params));
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not get next page url", e);
}
return new Page(getUrl(), continuation); // URL is ignored tho
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<CommentsInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (optCommentsDisabled.orElse(false)) {
return getInfoItemsPageForDisabledComments();
}
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
if (page == null || isNullOrEmpty(page.getId())) {
throw new IllegalArgumentException("Page doesn't have the continuation.");
}
final String ajaxResponse = makeAjaxRequest(page.getUrl());
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for comments", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("continuation", page.getId())
.done())
.getBytes(UTF_8);
final JsonObject ajaxJson = getJsonPostResponse("next", body, localization);
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectCommentsFrom(collector, ajaxJson);
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject ajaxJson) throws ParsingException {
final JsonArray contents;
try {
contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
} catch (final Exception e) {
//no comments
private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
@Nonnull final JsonObject ajaxJson) throws ParsingException {
final JsonArray onResponseReceivedEndpoints = ajaxJson.getArray(
"onResponseReceivedEndpoints");
final JsonObject commentsEndpoint = onResponseReceivedEndpoints.getObject(
onResponseReceivedEndpoints.size() - 1);
final String path;
if (commentsEndpoint.has("reloadContinuationItemsCommand")) {
path = "reloadContinuationItemsCommand.continuationItems";
} else if (commentsEndpoint.has("appendContinuationItemsAction")) {
path = "appendContinuationItemsAction.continuationItems";
} else {
// No comments
return;
}
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getArray(commentsEndpoint, path).clone();
} catch (final Exception e) {
// No comments
return;
}
final int index = contents.size() - 1;
if (contents.getObject(index).has("continuationItemRenderer")) {
contents.remove(index);
}
final List<Object> comments;
try {
comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
comments = JsonUtils.getValues(contents,
"commentThreadRenderer.comment.commentRenderer");
} catch (final Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
throw new ParsingException("Unable to get parse youtube comments", e);
}
for (final Object c : comments) {
if (c instanceof JsonObject) {
final CommentsInfoItemExtractor extractor =
new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser());
final CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(
(JsonObject) c, getUrl(), getTimeAgoParser());
collector.commit(extractor);
}
}
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
responseBody = YoutubeParsingHelper.unescapeDocument(response.responseBody());
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("videoId", getId())
.done())
.getBytes(UTF_8);
nextResponse = getJsonPostResponse("next", body, localization);
}
private String makeAjaxRequest(final String siteUrl) throws IOException, ReCaptchaException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept", singletonList("*/*"));
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", singletonList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", singletonList(ytClientName));
return getDownloader().get(siteUrl, requestHeaders, getExtractorLocalization()).responseBody();
}
private String getDataString(final Map<String, String> params) throws UnsupportedEncodingException {
final StringBuilder result = new StringBuilder();
boolean first = true;
for (final Map.Entry<String, String> entry : params.entrySet()) {
if (first) {
first = false;
} else {
result.append("&");
}
result.append(URLEncoder.encode(entry.getKey(), UTF_8));
result.append("=");
result.append(URLEncoder.encode(entry.getValue(), UTF_8));
}
return result.toString();
}
private String findValue(final String doc, final String start, final String end) {
int beginIndex = doc.indexOf(start);
// Start string was not found
if (beginIndex == -1) {
return null;
}
beginIndex = beginIndex + start.length();
final int endIndex = doc.indexOf(end, beginIndex);
// End string was not found
if (endIndex == -1) {
return null;
}
return doc.substring(beginIndex, endIndex);
}
@Override
public boolean isCommentsDisabled() {
public boolean isCommentsDisabled() throws ExtractionException {
// Check if commentsDisabled has to be initialized
if (!optCommentsDisabled.isPresent()) {
// Initialize commentsDisabled

91
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -21,7 +21,9 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
private final String url;
private final TimeAgoParser timeAgoParser;
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url, TimeAgoParser timeAgoParser) {
public YoutubeCommentsInfoItemExtractor(final JsonObject json,
final String url,
final TimeAgoParser timeAgoParser) {
this.json = json;
this.url = url;
this.timeAgoParser = timeAgoParser;
@ -37,7 +39,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@ -46,7 +48,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
public String getName() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
} catch (final Exception e) {
return EMPTY_STRING;
}
}
@ -55,7 +57,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
public String getTextualUploadDate() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
}
@ -64,7 +66,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public DateWrapper getUploadDate() throws ParsingException {
String textualPublishedTime = getTextualUploadDate();
if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) {
if (timeAgoParser != null && textualPublishedTime != null
&& !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime);
} else {
return null;
@ -72,33 +75,51 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
/**
* @implNote The method is parsing internally a localized string.<br>
* @implNote The method tries first to get the exact like count by using the accessibility data
* returned. But if the parsing of this accessibility data fails, the method parses internally
* a localized string.
* <br>
* <ul>
* <li>
* More than 1k likes will result in an inaccurate number
* </li>
* <li>
* This will fail for other languages than English.
* However as long as the Extractor only uses "en-GB"
* (as seen in {@link org.schabi.newpipe.extractor.services.youtube.YoutubeService#SUPPORTED_LANGUAGES})
* everything will work fine.
* </li>
* <li>More than 1k likes will result in an inaccurate number</li>
* <li>This will fail for other languages than English. However as long as the Extractor
* only uses "en-GB" (as seen in {@link
* org.schabi.newpipe.extractor.services.youtube.YoutubeService#getSupportedLocalizations})
* , everything will work fine.</li>
* </ul>
* <br>
* Consider using {@link #getTextualLikeCount()}
*/
@Override
public int getLikeCount() throws ParsingException {
// This may return a language dependent version, e.g. in German: 3,3 Mio
final String textualLikeCount = getTextualLikeCount();
// Try first to get the exact like count by using the accessibility data
final String likeCount;
try {
if (Utils.isBlank(textualLikeCount)) {
likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(json,
"actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer.accessibilityData.accessibilityData.label"));
} catch (final Exception e) {
// Use the approximate like count returned into the voteCount object
// This may return a language dependent version, e.g. in German: 3,3 Mio
final String textualLikeCount = getTextualLikeCount();
try {
if (Utils.isBlank(textualLikeCount)) {
return 0;
}
return (int) Utils.mixedNumberWordToLong(textualLikeCount);
} catch (final Exception i) {
throw new ParsingException(
"Unexpected error while converting textual like count to like count", i);
}
}
try {
if (Utils.isBlank(likeCount)) {
return 0;
}
return (int) Utils.mixedNumberWordToLong(textualLikeCount);
} catch (Exception e) {
throw new ParsingException("Unexpected error while converting textual like count to like count", e);
return Integer.parseInt(likeCount);
} catch (final Exception e) {
throw new ParsingException("Unexpected error while parsing like count as Integer", e);
}
}
@ -133,8 +154,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
return EMPTY_STRING;
}
return getTextFromObject(voteCountObj);
} catch (Exception e) {
throw new ParsingException("Could not get vote count", e);
} catch (final Exception e) {
throw new ParsingException("Could not get the vote count", e);
}
}
@ -148,9 +169,10 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
return EMPTY_STRING;
}
final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
// YouTube adds U+FEFF in some comments.
// eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
return Utils.removeUTF8BOM(commentText);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get comment text", e);
}
}
@ -159,7 +181,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
public String getCommentId() throws ParsingException {
try {
return JsonUtils.getString(json, "commentId");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get comment id", e);
}
}
@ -169,14 +191,16 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get author thumbnail", e);
}
}
@Override
public boolean isHeartedByUploader() throws ParsingException {
return json.has("creatorHeart");
final JsonObject commentActionButtonsRenderer = json.getObject("actionButtons")
.getObject("commentActionButtonsRenderer");
return commentActionButtonsRenderer.has("creatorHeart");
}
@Override
@ -185,15 +209,14 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
public boolean isUploaderVerified() {
// impossible to get this information from the mobile layout
return false;
return json.has("authorCommentBadge");
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
} catch (final Exception e) {
return EMPTY_STRING;
}
}
@ -201,10 +224,10 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getUploaderUrl() throws ParsingException {
try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) {
return "https://www.youtube.com/channel/" + JsonUtils.getString(json,
"authorEndpoint.browseEndpoint.browseId");
} catch (final Exception e) {
return EMPTY_STRING;
}
}
}

127
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -1,8 +1,10 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -11,6 +13,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@ -19,19 +22,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.net.URL;
import java.util.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
/**
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@ -58,12 +56,34 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final Response response = getResponse(url, getExtractorLocalization());
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
initialData = ajaxJson.getObject(3).getObject("response");
final Localization localization = getExtractorLocalization();
final URL url = stringToURL(getUrl());
final String mixPlaylistId = getId();
final String videoId = getQueryValue(url, "v");
final String playlistIndexString = getQueryValue(url, "index");
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
getExtractorContentCountry()).value("playlistId", mixPlaylistId);
if (videoId != null) {
jsonBody.value("videoId", videoId);
}
if (playlistIndexString != null) {
jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString));
}
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
headers, body, localization);
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
if (isNullOrEmpty(playlistData)) throw new ExtractionException(
"Could not get playlistData");
cookieValue = extractCookieValue(COOKIE_NAME, response);
}
@ -83,10 +103,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
} catch (final Exception e) {
try {
//fallback to thumbnail of current video. Always the case for channel mix
return getThumbnailUrlFromVideoId(
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
.getString("videoId"));
// Fallback to thumbnail of current video. Always the case for channel mix
return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
.getObject("watchEndpoint").getString("videoId"));
} catch (final Exception ignored) {
}
throw new ParsingException("Could not get playlist thumbnail", e);
@ -100,19 +119,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() {
//Youtube mix are auto-generated
// YouTube mixes are auto-generated by YouTube
return "";
}
@Override
public String getUploaderName() {
//Youtube mix are auto-generated by YouTube
// YouTube mixes are auto-generated by YouTube
return "YouTube";
}
@Override
public String getUploaderAvatarUrl() {
//Youtube mix are auto-generated by YouTube
// YouTube mixes are auto-generated by YouTube
return "";
}
@ -123,64 +142,81 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public long getStreamCount() {
// Auto-generated playlist always start with 25 videos and are endless
// Auto-generated playlists always start with 25 videos and are endless
return ListExtractor.ITEM_COUNT_INFINITE;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException,
ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray("contents"));
final Map<String, String> cookies = new HashMap<>();
cookies.put(COOKIE_NAME, cookieValue);
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies));
}
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
private Page getNextPageFrom(final JsonObject playlistJson,
final Map<String, String> cookies) throws IOException,
ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
.get(playlistJson.getArray("contents").size() - 1));
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
throw new ExtractionException("Could not extract next page url");
}
return getUrlFromNavigationEndpoint(
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+ "&pbj=1";
final JsonObject watchEndpoint = lastStream.getObject("playlistPanelVideoRenderer")
.getObject("navigationEndpoint").getObject("watchEndpoint");
final String playlistId = watchEndpoint.getString("playlistId");
final String videoId = watchEndpoint.getString("videoId");
final int index = watchEndpoint.getInt("index");
final String params = watchEndpoint.getString("params");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("videoId", videoId)
.value("playlistId", playlistId)
.value("playlistIndex", index)
.value("params", params)
.done())
.getBytes(UTF_8);
return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws ExtractionException, IOException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page url is empty or null");
throw new IllegalArgumentException("Page doesn't contain an URL");
}
if (!page.getCookies().containsKey(COOKIE_NAME)) {
throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing");
throw new IllegalArgumentException("Cookie '" + COOKIE_NAME + "' is missing");
}
final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());
final JsonObject playlistJson =
ajaxJson.getObject(3).getObject("response").getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist")
.getObject("playlist");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonObject playlistJson = ajaxJson.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
final JsonArray allStreams = playlistJson.getArray("contents");
// Sublist because youtube returns up to 24 previous streams in the mix
// Sublist because YouTube returns up to 24 previous streams in the mix
// +1 because the stream of "currentIndex" was already extracted in previous request
final List<Object> newStreams =
allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, newStreams);
return new InfoItemsPage<>(collector,
new Page(getNextPageUrlFrom(playlistJson), page.getCookies()));
return new InfoItemsPage<>(collector, getNextPageFrom(playlistJson, page.getCookies()));
}
private void collectStreamsFrom(
@Nonnull final StreamInfoItemsCollector collector,
@Nullable final List<Object> streams) {
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nullable final List<Object> streams) {
if (streams == null) {
return;
@ -193,7 +229,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final JsonObject streamInfo = ((JsonObject) stream)
.getObject("playlistPanelVideoRenderer");
if (streamInfo != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo,
timeAgoParser));
}
}
}
@ -204,7 +241,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
if (playlistId.startsWith("RDMM")) {
videoId = playlistId.substring(4);
} else if (playlistId.startsWith("RDCMUC")) {
throw new ParsingException("is channel mix");
throw new ParsingException("This playlist is a channel mix");
} else {
videoId = playlistId.substring(2);
}

186
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
@ -33,15 +34,18 @@ import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
public YoutubeMusicSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0];
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
+ youtubeMusicKeys[0];
final String params;
@ -67,17 +71,16 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
// @formatter:off
byte[] json = JsonWriter.string()
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en")
.value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.value("experimentsToken", EMPTY_STRING)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
@ -88,6 +91,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end()
.object("activePlayers").end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
.value("enableSafetyMode", false)
.end()
.end()
@ -103,11 +107,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers,
json));
try {
initialData = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
@ -121,20 +126,26 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer");
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents")
.getObject(0)
.getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) {
return "";
}
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) {
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
} else if (!showingResultsForRenderer.isEmpty()) {
return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query");
return JsonUtils.getString(showingResultsForRenderer,
"correctedQueryEndpoint.searchEndpoint.query");
} else {
return "";
}
@ -142,16 +153,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public boolean isCorrectedSearch() throws ParsingException {
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer");
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents")
.getObject(0)
.getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) {
return false;
}
JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
final boolean corrected = firstContent
.has("didYouMeanRenderer") || firstContent.has("showingResultsForRenderer");
return corrected;
return firstContent.has("didYouMeanRenderer")
|| firstContent.has("showingResultsForRenderer");
}
@Nonnull
@ -162,16 +176,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException, IOException {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents");
final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents");
Page nextPage = null;
for (Object content : contents) {
for (final Object content : contents) {
if (((JsonObject) content).has("musicShelfRenderer")) {
final JsonObject musicShelfRenderer = ((JsonObject) content).getObject("musicShelfRenderer");
final JsonObject musicShelfRenderer = ((JsonObject) content)
.getObject("musicShelfRenderer");
collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents"));
@ -183,14 +200,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
// @formatter:off
byte[] json = JsonWriter.string()
@ -227,16 +245,18 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(),
headers, json));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation");
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents")
.getObject("musicShelfContinuation");
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
final JsonArray continuations = musicShelfContinuation.getArray("continuations");
@ -244,31 +264,32 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
}
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) {
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector,
@Nonnull final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
for (final Object item : videos) {
final JsonObject info = ((JsonObject) item)
.getObject("musicResponsiveListItemRenderer", null);
if (info != null) {
final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING);
final String displayPolicy = info.getString("musicItemRendererDisplayPolicy",
EMPTY_STRING);
if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) {
continue; // no info about video URL available
continue; // No info about video URL available
}
final JsonObject flexColumnRenderer = info
.getArray("flexColumns")
final JsonObject flexColumnRenderer = info.getArray("flexColumns")
.getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer");
final JsonArray descriptionElements = flexColumnRenderer
.getObject("text")
final JsonArray descriptionElements = flexColumnRenderer.getObject("text")
.getArray("runs");
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
final String id = info.getObject("playlistItemData").getString("videoId");
final String id = info.getObject("playlistItemData")
.getString("videoId");
if (!isNullOrEmpty(id)) {
return "https://music.youtube.com/watch?v=" + id;
}
@ -277,8 +298,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String name = getTextFromObject(info.getArray("flexColumns")
.getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) {
return name;
}
@ -308,23 +331,34 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
if (searchType.equals(MUSIC_VIDEOS)) {
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items");
for (Object item : items) {
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer");
if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) {
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint"));
JsonArray items = info.getObject("menu").getObject("menuRenderer")
.getArray("items");
for (final Object item : items) {
final JsonObject menuNavigationItemRenderer =
((JsonObject) item).getObject(
"menuNavigationItemRenderer");
if (menuNavigationItemRenderer.getObject("icon")
.getString("iconType", EMPTY_STRING)
.equals("ARTIST")) {
return getUrlFromNavigationEndpoint(
menuNavigationItemRenderer
.getObject("navigationEndpoint"));
}
}
return null;
} else {
final JsonObject navigationEndpointHolder = info.getArray("flexColumns")
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
final JsonObject navigationEndpointHolder = info
.getArray("flexColumns")
.getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0);
if (!navigationEndpointHolder.has("navigationEndpoint")) return null;
if (!navigationEndpointHolder.has("navigationEndpoint"))
return null;
final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint"));
final String url = getUrlFromNavigationEndpoint(
navigationEndpointHolder.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
return url;
@ -366,13 +400,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@ -382,21 +418,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String name = getTextFromObject(info.getArray("flexColumns")
.getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) {
return name;
}
@ -405,7 +445,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint"));
final String url = getUrlFromNavigationEndpoint(info
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
return url;
}
@ -414,8 +455,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final String subscriberCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String subscriberCount = getTextFromObject(info
.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(subscriberCount)) {
try {
return Utils.mixedNumberWordToLong(subscriberCount);
@ -442,21 +485,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String name = getTextFromObject(info.getArray("flexColumns")
.getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) {
return name;
}
@ -509,7 +556,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
if (searchType.equals(MUSIC_ALBUMS)) {
return ITEM_COUNT_UNKNOWN;
}
final String count = descriptionElements.getObject(2).getString("text");
final String count = descriptionElements.getObject(2)
.getString("text");
if (!isNullOrEmpty(count)) {
if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100;
@ -525,17 +573,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
}
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
@Nullable
private Page getNextPageFrom(final JsonArray continuations)
throws IOException, ParsingException, ReCaptchaException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final JsonObject nextContinuationData = continuations.getObject(0)
.getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams + "&alt=json"
+ "&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]);
+ "&continuation=" + continuation + "&alt=json" + "&key="
+ YoutubeParsingHelper.getYoutubeMusicKey()[0]);
}
}

179
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -11,36 +11,28 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonArray initialAjaxJson;
private JsonObject initialData;
private JsonObject playlistInfo;
@ -49,27 +41,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // Show unavailable videos
.done())
.getBytes(UTF_8);
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = initialAjaxJson.getObject(1).getObject("response");
initialData = getJsonPostResponse("browse", body, localization);
YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo();
}
private JsonObject getUploaderInfo() throws ParsingException {
final JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
final JsonArray items = initialData.getObject("sidebar")
.getObject("playlistSidebarRenderer").getArray("items");
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
JsonObject videoOwner = items.getObject(1)
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
// we might want to create a loop here instead of using duplicated code
videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
videoOwner = items.getObject(items.size())
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
@ -78,9 +78,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject getPlaylistInfo() throws ParsingException {
try {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items")
.getObject(0).getObject("playlistSidebarPrimaryInfoRenderer");
} catch (Exception e) {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer")
.getArray("items").getObject(0)
.getObject("playlistSidebarPrimaryInfoRenderer");
} catch (final Exception e) {
throw new ParsingException("Could not get PlaylistInfo", e);
}
}
@ -120,7 +121,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderUrl() throws ParsingException {
try {
return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader url", e);
}
}
@ -129,7 +130,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader name", e);
}
}
@ -140,7 +141,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e);
}
}
@ -155,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try {
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get video count from playlist", e);
}
}
@ -184,18 +185,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
final JsonArray contents = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
if (((JsonObject) segment).getObject("playlistSegmentRenderer")
.has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment)
.getObject("playlistSegmentRenderer").getObject("videoList")
.getObject("playlistVideoListRenderer").getArray("contents"));
}
}
@ -212,20 +214,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getObject(0).getObject("appendContinuationItemsAction")
.getArray("continuationItems");
collectStreamsFrom(collector, continuation);
@ -233,7 +237,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
private Page getNextPageFrom(final JsonArray contents) throws IOException,
ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
@ -246,25 +251,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder()
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);
return new Page(
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
body);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
} else {
return null;
}
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object video : videos) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video)
.getObject("playlistVideoRenderer"), timeAgoParser) {
@Override
public long getViewCount() {
return -1;
@ -273,81 +279,4 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
}
private void collectTrailerFrom(final StreamInfoItemsCollector collector,
final JsonObject segment) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public String getName() throws ParsingException {
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("title"));
}
@Override
public String getUrl() throws ParsingException {
return YoutubeStreamLinkHandlerFactory.getInstance()
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
.getUrl();
}
@Override
public String getThumbnailUrl() {
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
}
@Override
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
@Override
public long getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("segmentAnnotation")).split("•")[0]);
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderName() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderUrl();
}
@Override
public boolean isUploaderVerified() {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
});
}
}

166
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
@ -17,11 +18,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -49,17 +49,37 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchExtractor extends SearchExtractor {
private JsonObject initialData;
public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
public YoutubeSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String query = super.getSearchString();
final Localization localization = getExtractorLocalization();
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
// Get the search parameter of the request
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
final String params;
if (!isNullOrEmpty(contentFilters)) {
final String searchType = contentFilters.get(0);
params = getSearchParameter(searchType);
} else {
params = "";
}
initialData = ajaxJson.getObject(1).getObject("response");
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("query", query);
if (!isNullOrEmpty(params)) {
jsonBody.value("params", params);
}
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
initialData = getJsonPostResponse("search", body, localization);
}
@Nonnull
@ -77,11 +97,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getObject("itemSectionRenderer");
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) {
return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query");
return JsonUtils.getString(didYouMeanRenderer,
"correctedQueryEndpoint.searchEndpoint.query");
} else if (showingResultsForRenderer != null) {
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
} else {
@ -103,7 +125,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"));
.getObject("primaryContents").getObject("sectionListRenderer")
.getArray("contents"));
}
@Nonnull
@ -111,20 +134,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
final JsonArray sections = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents");
Page nextPage = null;
for (final Object section : sections) {
if (((JsonObject) section).has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
final JsonObject itemSectionRenderer = ((JsonObject) section)
.getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
} else if (((JsonObject) section).has("continuationItemRenderer")) {
nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer"));
nextPage = getNextPageFrom(((JsonObject) section)
.getObject("continuationItemRenderer"));
}
}
@ -132,98 +156,70 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Localization localization = getExtractorLocalization();
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
if (page.getId() == null) {
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
// @formatter:off
final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("continuation", page.getId())
.done())
.getBytes(UTF_8);
// @formatter:on
final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
final String responseBody = getValidJsonResponseBody(getDownloader().post(
page.getUrl(), new HashMap<>(), json));
collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
final JsonArray continuations = itemSectionContinuation.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
} else {
// @formatter:off
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("utcOffsetMinutes", 0)
.end()
.object("request").end()
.object("user").end()
.end()
.value("continuation", page.getId())
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList(this.getUrl()));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents");
collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer")));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction")
.getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1)
.getObject("continuationItemRenderer")));
}
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray contents) throws NothingFoundException, ParsingException {
private void collectStreamsFrom(final InfoItemsSearchCollector collector,
final JsonArray contents) throws NothingFoundException,
ParsingException {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object content : contents) {
for (final Object content : contents) {
final JsonObject item = (JsonObject) content;
if (item.has("backgroundPromoRenderer")) {
throw new NothingFoundException(getTextFromObject(
item.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (item.has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(item.getObject("videoRenderer"), timeAgoParser));
collector.commit(new YoutubeStreamInfoItemExtractor(item
.getObject("videoRenderer"), timeAgoParser));
} else if (item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(item.getObject("channelRenderer")));
collector.commit(new YoutubeChannelInfoItemExtractor(item
.getObject("channelRenderer")));
} else if (item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(item.getObject("playlistRenderer")));
collector.commit(new YoutubePlaylistInfoItemExtractor(item
.getObject("playlistRenderer")));
}
}
}
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
}
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuationItemRenderer)) {
return null;
}
@ -231,7 +227,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
return new Page(url, token);
}

779
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

File diff suppressed because it is too large Load Diff

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -22,6 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -38,28 +39,32 @@ import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData;
public YoutubeTrendingExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId) {
public YoutubeTrendingExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final String kioskId) {
super(service, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "?pbj=1&gl="
+ getExtractorContentCountry().getCountryCode();
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
// @formatter:off
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("browseId", "FEtrending")
.done())
.getBytes(UTF_8);
// @formatter:on
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
}
@Override
@ -89,15 +94,17 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents");
JsonArray itemSectionRenderers = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents");
for (Object itemSectionRenderer : itemSectionRenderers) {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
for (final Object itemSectionRenderer : itemSectionRenderers) {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
for (final Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View File

@ -16,7 +16,7 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(String id) {
return "https://m.youtube.com/watch?v=" + id;
return "https://www.youtube.com/watch?v=" + id;
}
@Override

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -3,11 +3,13 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import javax.annotation.Nonnull;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -25,24 +27,31 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
private static final String SEARCH_URL = "https://www.youtube.com/results?search_query=";
private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q=";
@Nonnull
public static YoutubeSearchQueryHandlerFactory getInstance() {
return new YoutubeSearchQueryHandlerFactory();
}
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
public String getUrl(final String searchString,
@Nonnull final List<String> contentFilters,
final String sortFilter) throws ParsingException {
try {
if (!contentFilters.isEmpty()) {
switch (contentFilters.get(0)) {
final String contentFilter = contentFilters.get(0);
switch (contentFilter) {
case ALL:
default:
break;
case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
+ "&sp=EgIQAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
+ "&sp=EgIQAg%253D%253D";
case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
+ "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
@ -53,7 +62,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
}
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}
}
@ -69,7 +78,28 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
MUSIC_VIDEOS,
MUSIC_ALBUMS,
MUSIC_PLAYLISTS
// MUSIC_ARTISTS
// MUSIC_ARTISTS
};
}
@Nonnull
public static String getSearchParameter(final String contentFilter) {
if (isNullOrEmpty(contentFilter)) return "";
switch (contentFilter) {
case VIDEOS:
return "EgIQAQ%3D%3D";
case CHANNELS:
return "EgIQAg%3D%3D";
case PLAYLISTS:
return "EgIQAw%3D%3D";
case ALL:
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
default:
return "";
}
}
}

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.utils;
import javax.annotation.Nonnull;
public class StringUtils {

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -59,7 +59,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -72,7 +72,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -86,7 +86,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCPWXIOPK-9myzek6jHR5yrg");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -100,7 +100,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -115,7 +115,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCoaO4U_p7G7AwalqSbGCZOA");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -129,7 +129,7 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCpExuV8qJMfCaSQNL1YG6bQ");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
} catch (final AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
@ -619,7 +619,7 @@ public class YoutubeChannelExtractorTest {
public void testMoreRelatedItems() {
try {
defaultTestMoreItems(extractor);
} catch (Throwable ignored) {
} catch (final Throwable ignored) {
return;
}
@ -667,4 +667,3 @@ public class YoutubeChannelExtractorTest {
}
}
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java
View File

@ -40,10 +40,10 @@ public class YoutubeChannelLocalizationTest {
testLocalizationsFor("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg");
}
private void testLocalizationsFor(String channelUrl) throws Exception {
private void testLocalizationsFor(final String channelUrl) throws Exception {
final List<Localization> supportedLocalizations = YouTube.getSupportedLocalizations();
// final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
// final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
final Map<Localization, List<StreamInfoItem>> results = new LinkedHashMap<>();
for (Localization currentLocalization : supportedLocalizations) {
@ -55,7 +55,7 @@ public class YoutubeChannelLocalizationTest {
extractor.forceLocalization(currentLocalization);
extractor.fetchPage();
itemsPage = defaultTestRelatedItems(extractor);
} catch (Throwable e) {
} catch (final Throwable e) {
System.out.println("[!] " + currentLocalization + " → failed");
throw e;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java → extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java
View File

@ -11,7 +11,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class YouTubeCommentsLinkHandlerFactoryTest {
public class YoutubeCommentsLinkHandlerFactoryTest {
private static YoutubeCommentsLinkHandlerFactory linkHandler;

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java
View File

@ -93,6 +93,5 @@ public class YoutubeFeedExtractorTest {
.getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
extractor.fetchPage();
}
}
}

117
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -1,23 +1,16 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonWriter;
import org.hamcrest.MatcherAssert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -32,12 +25,11 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
@RunWith(Suite.class)
@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
public class YoutubeMixPlaylistExtractorTest {
public static final String PBJ = "&pbj=1";
private static final String VIDEO_ID = "_AzeUSL9lZc";
private static final String VIDEO_TITLE =
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
@ -46,6 +38,7 @@ public class YoutubeMixPlaylistExtractorTest {
private static YoutubeMixPlaylistExtractor extractor;
@Ignore("Test broken, video was blocked by SME and is only available in Japan")
public static class Mix {
@BeforeClass
@ -55,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RD" + VIDEO_ID);
extractor.fetchPage();
}
@ -89,9 +82,16 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
+ PBJ, dummyCookie));
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RD" + VIDEO_ID)
.value("params", "OAE%3D")
.done())
.getBytes(UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -101,14 +101,14 @@ public class YoutubeMixPlaylistExtractorTest {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
// Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
for (final StreamInfoItem item : streams.getItems()) {
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
@ -124,10 +124,10 @@ public class YoutubeMixPlaylistExtractorTest {
}
}
@Ignore
@Ignore("Test broken, video was removed by the uploader")
public static class MixWithIndex {
private static final String INDEX = "&index=13";
private static final int INDEX = 13;
private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk";
@BeforeClass
@ -137,9 +137,8 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13
+ "&list=RD" + VIDEO_ID + "&index=" + INDEX);
extractor.fetchPage();
}
@ -167,9 +166,17 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RD" + VIDEO_ID)
.value("playlistIndex", INDEX)
.value("params", "OAE%3D")
.done())
.getBytes(UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -179,13 +186,13 @@ public class YoutubeMixPlaylistExtractorTest {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
// Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
for (final StreamInfoItem item : streams.getItems()) {
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
@ -201,6 +208,7 @@ public class YoutubeMixPlaylistExtractorTest {
}
}
@Ignore("Test broken")
public static class MyMix {
@BeforeClass
@ -210,9 +218,8 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID);
extractor.fetchPage();
}
@ -243,9 +250,16 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams =
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RDMM" + VIDEO_ID)
.value("params", "OAE%3D")
.done())
.getBytes(UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -255,14 +269,14 @@ public class YoutubeMixPlaylistExtractorTest {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();
//Should work infinitely, but for testing purposes only 3 times
// Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
for (final StreamInfoItem item : streams.getItems()) {
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
// assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}
@ -288,11 +302,12 @@ public class YoutubeMixPlaylistExtractorTest {
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
}
@Ignore
@Test(expected = IllegalArgumentException.class)
public void getPageEmptyUrl() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RD" + VIDEO_ID);
extractor.fetchPage();
extractor.getPage(new Page(""));
}
@ -300,8 +315,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test(expected = ExtractionException.class)
public void invalidVideoId() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + "abcde"
+ "&list=RD" + "abcde");
extractor.fetchPage();
extractor.getName();
}
@ -321,9 +336,8 @@ public class YoutubeMixPlaylistExtractorTest {
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
extractor.fetchPage();
}
@ -350,9 +364,16 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID_OF_CHANNEL)
.value("playlistId", "RDCM" + CHANNEL_ID)
.value("params", "OAE%3D")
.done())
.getBytes(UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@ -25,15 +25,15 @@ public class YoutubeParsingHelperTest {
}
@Test
public void testIsHardcodedClientVersionValid() throws IOException, ExtractionException {
assertTrue("Hardcoded client version is not valid anymore",
YoutubeParsingHelper.isHardcodedClientVersionValid());
public void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException {
assertTrue("Hardcoded client version and key are not valid anymore",
YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid());
}
@Test
public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException {
assertTrue("Hardcoded YouTube Music keys are not valid anymore",
YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid());
YoutubeParsingHelper.isHardcodedYoutubeMusicKeyValid());
}
@Test
@ -44,7 +44,7 @@ public class YoutubeParsingHelperTest {
}
@Test
public void testConvertFromGoogleCacheUrl() throws ParsingException {
public void testConvertFromGoogleCacheUrl() {
assertEquals("https://mohfw.gov.in/",
YoutubeParsingHelper.extractCachedUrlIfNeeded("https://webcache.googleusercontent.com/search?q=cache:https://mohfw.gov.in/"));
assertEquals("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html",

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -3,9 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -13,11 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -38,9 +30,6 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
/**
* Test for {@link YoutubePlaylistExtractor}
*/
@RunWith(Suite.class)
@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class,
LearningPlaylist.class, ContinuationsTests.class})
public class YoutubePlaylistExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/playlist/";
@ -61,7 +50,6 @@ public class YoutubePlaylistExtractorTest {
}
@Test(expected = ContentNotAvailableException.class)
@Ignore("Broken, now invalid playlists redirect to youtube homepage")
public void invalidId() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java
View File

@ -17,7 +17,7 @@ import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
// Doesn't work with mocks. Makes request with different `dataToSend` i think
// Doesn't work with mocks. Makes request with different `dataToSend` I think
public class YoutubeMusicSearchExtractorTest {
public static class MusicSongs extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.InfoItem;
@ -272,13 +273,14 @@ public class YoutubeSearchExtractorTest {
urlTexts
));
}
// testMoreRelatedItems is broken because a video has no duration shown
@Override public void testMoreRelatedItems() { }
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedOriginalUrlContains() throws Exception { return "youtube.com/results?search_query=" + QUERY; }
}
public static class ChannelVerified extends DefaultSearchExtractorTest {
@ -318,5 +320,4 @@ public class YoutubeSearchExtractorTest {
assertTrue(verified);
}
}
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -28,6 +29,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ageRestricted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -54,10 +56,10 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Override public long expectedDislikeCountAtLeast() { return 38000; }
@Override public boolean expectedHasRelatedItems() { return false; } // no related videos (!)
@Override public int expectedAgeLimit() { return 18; }
@Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos
@Override public String expectedCategory() { return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override
public List<String> expectedTags() {

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@ -1,18 +1,19 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import javax.annotation.Nullable;
@ -21,7 +22,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamLinkHandlerFactory}
*/
@Ignore("Video is not available in specific countries. Someone else has to generate mocks")
public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "T4XJQO3qol8";
@ -31,6 +32,8 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -59,5 +62,4 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
@Override public List<String> expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); }
@Override public String expectedCategory() { return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -17,10 +17,7 @@ import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentExcepti
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.*;
import java.io.IOException;
import java.net.MalformedURLException;
@ -66,6 +63,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@ -122,6 +120,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -165,6 +164,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -218,6 +218,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -255,6 +256,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsTagesschau"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -316,6 +318,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -386,6 +389,7 @@ public class YoutubeStreamExtractorDefaultTest {
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast"));
extractor = YouTube.getStreamExtractor(URL);
YoutubeStreamExtractor.resetDeobfuscationCode();
extractor.fetchPage();
}
@ -435,6 +439,7 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
@ -454,6 +459,7 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -30,6 +30,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "live"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
@ -37,7 +38,6 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@Override
@Test
@Ignore("When visiting website it shows 'Lofi Girl', unknown why it's different in tests")
public void testUploaderName() throws Exception {
super.testUploaderName();
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -28,6 +29,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unlisted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();

200
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

231
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

249
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

79
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

243
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

249
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

79
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

231
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

249
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

117
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

372
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

347
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

232
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

249
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

252
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_4.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_5.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

252
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_4.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_5.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_6.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_7.json
View File

File diff suppressed because one or more lines are too long

238
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_8.json
View File

@ -1,62 +1,244 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid",
"httpMethod": "POST",
"url": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"Cookie": [
"CONSENT\u003dPENDING+506"
"Origin": [
"https://www.youtube.com"
],
"X-YouTube-Client-Name": [
"1"
],
"Referer": [
"https://www.youtube.com"
],
"X-YouTube-Client-Version": [
"2.20200214.04.00"
"2.20210728.00.00"
],
"Content-Type": [
"application/json"
]
},
"dataToSend": [
123,
34,
98,
114,
111,
119,
115,
101,
73,
100,
34,
58,
34,
68,
79,
69,
83,
78,
84,
45,
69,
88,
73,
83,
84,
34,
44,
34,
99,
111,
110,
116,
101,
120,
116,
34,
58,
123,
34,
99,
108,
105,
101,
110,
116,
34,
58,
123,
34,
104,
108,
34,
58,
34,
101,
110,
45,
71,
66,
34,
44,
34,
103,
108,
34,
58,
34,
71,
66,
34,
44,
34,
99,
108,
105,
101,
110,
116,
78,
97,
109,
101,
34,
58,
34,
87,
69,
66,
34,
44,
34,
99,
108,
105,
101,
110,
116,
86,
101,
114,
115,
105,
111,
110,
34,
58,
34,
50,
46,
50,
48,
50,
49,
48,
55,
50,
56,
46,
48,
48,
46,
48,
48,
34,
125,
44,
34,
117,
115,
101,
114,
34,
58,
123,
34,
108,
111,
99,
107,
101,
100,
83,
97,
102,
101,
116,
121,
77,
111,
100,
101,
34,
58,
102,
97,
108,
115,
101,
125,
125,
44,
34,
112,
97,
114,
97,
109,
115,
34,
58,
34,
69,
103,
90,
50,
97,
87,
82,
108,
98,
51,
77,
37,
51,
68,
34,
125
],
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 404,
"responseCode": 400,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
"private"
],
"content-type": [
"text/html; charset\u003dutf-8"
"application/json; charset\u003dUTF-8"
],
"date": [
"Sat, 03 Jul 2021 11:29:58 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
],
"pragma": [
"no-cache"
"Fri, 30 Jul 2021 17:13:39 GMT"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dotZ2jZ94DRk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
"vary": [
"Origin",
"X-Origin",
"Referer"
],
"x-content-type-options": [
"nosniff"
@ -68,7 +250,7 @@
"0"
]
},
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"n/vBxRiZa1jxbE1/ttyVwQ\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
"responseBody": "{\n \"error\": {\n \"code\": 400,\n \"message\": \"Request contains an invalid argument.\",\n \"errors\": [\n {\n \"message\": \"Request contains an invalid argument.\",\n \"domain\": \"global\",\n \"reason\": \"badRequest\"\n }\n ],\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n",
"latestUrl": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
}
}

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notSupported/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notSupported/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/random/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

251
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/random/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

199
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/empty/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/empty/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/empty/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

199
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/hearted/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/hearted/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/hearted/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

199
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/likes/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/likes/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/likes/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

205
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/localized_vote_count/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/localized_vote_count/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/localized_vote_count/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

199
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/pinned/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/pinned/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/pinned/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

199
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

288
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_3.json
View File

File diff suppressed because one or more lines are too long

289
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_4.json
View File

File diff suppressed because one or more lines are too long

315
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/thomas/generated_mock_5.json Normal file
View File

File diff suppressed because one or more lines are too long

8
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

2
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/notAvailable/generated_mock_0.json
View File

@ -26,7 +26,7 @@
"text/html; charset\u003dUTF-8"
],
"date": [
"Sat, 03 Jul 2021 11:29:58 GMT"
"Sun, 04 Jul 2021 16:47:38 GMT"
],
"server": [
"YouTube RSS Feeds server"

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

211
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/channelMix/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

252
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/channelMix/generated_mock_1.json
View File

File diff suppressed because one or more lines are too long

270
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/channelMix/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/invalid/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

230
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/invalid/generated_mock_1.json
View File

@ -1,65 +1,230 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/watch?v\u003dabcde\u0026list\u003dRDabcde\u0026pbj\u003d1",
"httpMethod": "POST",
"url": "https://www.youtube.com/youtubei/v1/next?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"Cookie": [
"CONSENT\u003dPENDING+385"
"Origin": [
"https://www.youtube.com"
],
"X-YouTube-Client-Name": [
"1"
],
"Referer": [
"https://www.youtube.com"
],
"X-YouTube-Client-Version": [
"2.20200214.04.00"
"2.20210728.00.00"
]
},
"dataToSend": [
123,
34,
112,
108,
97,
121,
108,
105,
115,
116,
73,
100,
34,
58,
34,
82,
68,
97,
98,
99,
100,
101,
34,
44,
34,
99,
111,
110,
116,
101,
120,
116,
34,
58,
123,
34,
99,
108,
105,
101,
110,
116,
34,
58,
123,
34,
104,
108,
34,
58,
34,
101,
110,
45,
71,
66,
34,
44,
34,
103,
108,
34,
58,
34,
71,
66,
34,
44,
34,
99,
108,
105,
101,
110,
116,
78,
97,
109,
101,
34,
58,
34,
87,
69,
66,
34,
44,
34,
99,
108,
105,
101,
110,
116,
86,
101,
114,
115,
105,
111,
110,
34,
58,
34,
50,
46,
50,
48,
50,
49,
48,
55,
50,
56,
46,
48,
48,
46,
48,
48,
34,
125,
44,
34,
117,
115,
101,
114,
34,
58,
123,
34,
108,
111,
99,
107,
101,
100,
83,
97,
102,
101,
116,
121,
77,
111,
100,
101,
34,
58,
102,
97,
108,
115,
101,
125,
125,
44,
34,
118,
105,
100,
101,
111,
73,
100,
34,
58,
34,
97,
98,
99,
100,
101,
34,
125
],
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseCode": 500,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
],
"content-disposition": [
"attachment"
"private"
],
"content-type": [
"application/json; charset\u003dutf-8"
"application/json; charset\u003dUTF-8"
],
"date": [
"Sat, 03 Jul 2021 11:29:31 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
],
"pragma": [
"no-cache"
"Sun, 01 Aug 2021 15:15:10 GMT"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003d6I04qC_jDQY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
"vary": [
"Origin",
"X-Origin",
"Referer"
],
"x-content-type-options": [
"nosniff"
@ -67,14 +232,11 @@
"x-frame-options": [
"SAMEORIGIN"
],
"x-spf-response-type": [
"multipart"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r4---sn-4g5ednld.googlevideo.com\\/generate_204\",\"https:\\/\\/r4---sn-4g5ednld.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23968386,23891346,23885487,23966208,24058240,23975059,23884386,24012512,24056839,24023960,23804281,24030040,24045412,23999405,24045470,23821390,24049575,24060173,23946420,24037794,24042870,24003103,24059009,1714254,24007246,23986021,23991736,24034977,24057008,23934970,23983296,24003105,23983814,24043960,23891344,24011363,24056265,24059522,24062574,24058128,24058293,24049577,24058780,24038425,24049569,23857950,23996830,23940237,23890959,24049820,24044124,23973490,23744176,24045469,23918597,24049567,23877023,24049573,23998056,24058812,23882685,24027649,23944779,24004644,23974595,24036948,24053866,24001373,24059897,24058380,24052245,24058861,24063702\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x860a6041dbed4108\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210526\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwjH_46W5sbxAhUVD-AKHU6hBdo\u003d\",\"frameworkUpdates\":{\"entityBatchUpdate\":{\"mutations\":[{\"entityKey\":\"EgcKBWFiY2RlIPYBKAE%3D\",\"type\":\"ENTITY_MUTATION_TYPE_REPLACE\",\"payload\":{\"offlineabilityEntity\":{\"key\":\"EgcKBWFiY2RlIPYBKAE%3D\",\"accessState\":\"OFFLINEABILITY_FEATURE_ACCESS_STATE_UNKNOWN\"}}}],\"timestamp\":{\"seconds\":\"1625311771\",\"nanos\":92746917}}}}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtEZDdTSEszdmlXSSiblIGHBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbXk3WU5od2t4OV9Gc0V6TXdFQ0FTQ1U2c2Z2Z3xBQ3Jtc0trRm9jNTQ3UXUydGFVdUJsYTVqQVptaUVsUHk5NmZTaTVRYjlQam5BVG1iaVVZLUs1aGlJQWpPdUtCSmdxVWg3VUVfQWdYR3Y0eTZ0SjFHUHF0YkI5OEhrbkRhU0tIWFUwUTh1Vm0ta09iVVVIeGVpbw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dabcde\\u0026list\\u003dRDabcde\",\"endpoint\": {\"clickTrackingParams\":\"IhMImKCOlubG8QIVjofeCh0Zow5zMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dabcde\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"abcde\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\": 0.0 }}}]\r\n",
"latestUrl": "https://www.youtube.com/watch?v\u003dabcde\u0026list\u003dRDabcde\u0026pbj\u003d1"
"responseBody": "{\n \"error\": {\n \"code\": 500,\n \"message\": \"Internal error encountered.\",\n \"errors\": [\n {\n \"message\": \"Internal error encountered.\",\n \"domain\": \"global\",\n \"reason\": \"backendError\"\n }\n ],\n \"status\": \"INTERNAL\"\n }\n}\n",
"latestUrl": "https://www.youtube.com/youtubei/v1/next?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
}
}

80
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/invalid/generated_mock_2.json
View File

File diff suppressed because one or more lines are too long

200
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/mix/generated_mock_0.json
View File

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More