be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Fix checkstyle issues

This commit is contained in:
Stypox 2022-03-18 15:09:06 +01:00 committed by litetex
parent 9dc17cd1ca
commit 740a37a2de
27 changed files with 684 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -1,14 +1,22 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA_OPUS;
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.schabi.newpipe.extractor.MediaFormat.*;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.*;
public class ItagItem { public class ItagItem {
/** /**
* List can be found here https://github.com/ytdl-org/youtube-dl/blob/9fc5eafb8e384453a49f7cfe73147be491f0b19d/youtube_dl/extractor/youtube.py#L1071 * List can be found here
* https://github.com/ytdl-org/youtube-dl/blob/9fc5eafb8e384453a49f7cfe73147be491f0b19d/youtube_dl/extractor/youtube.py#L1071
*/ */
private static final ItagItem[] ITAG_LIST = { private static final ItagItem[] ITAG_LIST = {
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
@ -79,8 +87,8 @@ public class ItagItem {
// Utils // Utils
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
public static boolean isSupported(int itag) { public static boolean isSupported(final int itag) {
for (ItagItem item : ITAG_LIST) { for (final ItagItem item : ITAG_LIST) {
if (itag == item.id) { if (itag == item.id) {
return true; return true;
} }
@ -88,8 +96,8 @@ public class ItagItem {
return false; return false;
} }
public static ItagItem getItag(int itagId) throws ParsingException { public static ItagItem getItag(final int itagId) throws ParsingException {
for (ItagItem item : ITAG_LIST) { for (final ItagItem item : ITAG_LIST) {
if (itagId == item.id) { if (itagId == item.id) {
return item; return item;
} }
@ -110,7 +118,10 @@ public class ItagItem {
/** /**
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30. * Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
*/ */
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) { public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final String resolution) {
this.id = id; this.id = id;
this.itagType = type; this.itagType = type;
this.mediaFormat = format; this.mediaFormat = format;
@ -123,7 +134,11 @@ public class ItagItem {
* *
* @param resolution string that will be used in the frontend * @param resolution string that will be used in the frontend
*/ */
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) { public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final String resolution,
final int fps) {
this.id = id; this.id = id;
this.itagType = type; this.itagType = type;
this.mediaFormat = format; this.mediaFormat = format;
@ -131,7 +146,10 @@ public class ItagItem {
this.fps = fps; this.fps = fps;
} }
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) { public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final int avgBitrate) {
this.id = id; this.id = id;
this.itagType = type; this.itagType = type;
this.mediaFormat = format; this.mediaFormat = format;
@ -170,7 +188,7 @@ public class ItagItem {
return bitrate; return bitrate;
} }
public void setBitrate(int bitrate) { public void setBitrate(final int bitrate) {
this.bitrate = bitrate; this.bitrate = bitrate;
} }
@ -178,7 +196,7 @@ public class ItagItem {
return width; return width;
} }
public void setWidth(int width) { public void setWidth(final int width) {
this.width = width; this.width = width;
} }
@ -186,7 +204,7 @@ public class ItagItem {
return height; return height;
} }
public void setHeight(int height) { public void setHeight(final int height) {
this.height = height; this.height = height;
} }
@ -194,7 +212,7 @@ public class ItagItem {
return initStart; return initStart;
} }
public void setInitStart(int initStart) { public void setInitStart(final int initStart) {
this.initStart = initStart; this.initStart = initStart;
} }
@ -202,7 +220,7 @@ public class ItagItem {
return initEnd; return initEnd;
} }
public void setInitEnd(int initEnd) { public void setInitEnd(final int initEnd) {
this.initEnd = initEnd; this.initEnd = initEnd;
} }
@ -210,7 +228,7 @@ public class ItagItem {
return indexStart; return indexStart;
} }
public void setIndexStart(int indexStart) { public void setIndexStart(final int indexStart) {
this.indexStart = indexStart; this.indexStart = indexStart;
} }
@ -218,7 +236,7 @@ public class ItagItem {
return indexEnd; return indexEnd;
} }
public void setIndexEnd(int indexEnd) { public void setIndexEnd(final int indexEnd) {
this.indexEnd = indexEnd; this.indexEnd = indexEnd;
} }
@ -226,7 +244,7 @@ public class ItagItem {
return quality; return quality;
} }
public void setQuality(String quality) { public void setQuality(final String quality) {
this.quality = quality; this.quality = quality;
} }
@ -234,7 +252,7 @@ public class ItagItem {
return codec; return codec;
} }
public void setCodec(String codec) { public void setCodec(final String codec) {
this.codec = codec; this.codec = codec;
} }
} }

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java
View File

@ -19,7 +19,7 @@ import javax.annotation.Nonnull;
* This class handling fetching the JavaScript file in order to allow other classes to extract the * This class handling fetching the JavaScript file in order to allow other classes to extract the
* needed functions. * needed functions.
*/ */
public class YoutubeJavaScriptExtractor { public final class YoutubeJavaScriptExtractor {
private static final String HTTPS = "https:"; private static final String HTTPS = "https:";
private static String cachedJavaScriptCode; private static String cachedJavaScriptCode;
@ -81,9 +81,10 @@ public class YoutubeJavaScriptExtractor {
final String hashPattern = "player\\\\\\/([a-z0-9]{8})\\\\\\/"; final String hashPattern = "player\\\\\\/([a-z0-9]{8})\\\\\\/";
final String hash = Parser.matchGroup1(hashPattern, iframeContent); final String hash = Parser.matchGroup1(hashPattern, iframeContent);
return String.format("https://www.youtube.com/s/player/%s/player_ias.vflset/en_US/base.js", hash); return String.format(
"https://www.youtube.com/s/player/%s/player_ias.vflset/en_US/base.js", hash);
} catch (final Exception i) { } } catch (final Exception ignored) {
}
throw new ParsingException("Iframe API did not provide YouTube player js url"); throw new ParsingException("Iframe API did not provide YouTube player js url");
} }
@ -109,8 +110,8 @@ public class YoutubeJavaScriptExtractor {
} }
} }
} }
} catch (final Exception ignored) {
} catch (final Exception i) { } }
throw new ParsingException("Embedded info did not provide YouTube player js url"); throw new ParsingException("Embedded info did not provide YouTube player js url");
} }

200
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1,5 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
@ -10,7 +17,11 @@ import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*; import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
@ -28,18 +39,18 @@ import java.time.LocalDate;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Created by Christian Schabesberger on 02.03.16. * Created by Christian Schabesberger on 02.03.16.
* *
@ -60,7 +71,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class YoutubeParsingHelper { public final class YoutubeParsingHelper {
private YoutubeParsingHelper() { private YoutubeParsingHelper() {
} }
@ -99,10 +110,10 @@ public class YoutubeParsingHelper {
"https://www.youtube.com/feeds/videos.xml?channel_id="; "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static boolean isGoogleURL(String url) { private static boolean isGoogleURL(final String url) {
url = extractCachedUrlIfNeeded(url); final String cachedUrl = extractCachedUrlIfNeeded(url);
try { try {
final URL u = new URL(url); final URL u = new URL(cachedUrl);
final String host = u.getHost(); final String host = u.getHost();
return host.startsWith("google.") return host.startsWith("google.")
|| host.startsWith("m.google.") || host.startsWith("m.google.")
@ -442,7 +453,10 @@ public class YoutubeParsingHelper {
private static void extractClientVersionAndKey() throws IOException, ExtractionException { private static void extractClientVersionAndKey() throws IOException, ExtractionException {
// Don't extract the client version and the InnerTube key if it has been already extracted // Don't extract the client version and the InnerTube key if it has been already extracted
if (keyAndVersionExtracted) return; if (keyAndVersionExtracted) {
return;
}
// Don't provide a search term in order to have a smaller response // Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query=&ucbcb=1"; final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
@ -460,8 +474,8 @@ public class YoutubeParsingHelper {
final JsonArray params = s.getArray("params"); final JsonArray params = s.getArray("params");
for (final Object param : params) { for (final Object param : params) {
final JsonObject p = (JsonObject) param; final JsonObject p = (JsonObject) param;
final String key = p.getString("key"); final String paramKey = p.getString("key");
if (key != null && key.equals("cver")) { if (paramKey != null && paramKey.equals("cver")) {
clientVersion = p.getString("value"); clientVersion = p.getString("value");
} }
} }
@ -471,8 +485,8 @@ public class YoutubeParsingHelper {
final JsonArray params = s.getArray("params"); final JsonArray params = s.getArray("params");
for (final Object param : params) { for (final Object param : params) {
final JsonObject p = (JsonObject) param; final JsonObject p = (JsonObject) param;
final String key = p.getString("key"); final String paramKey = p.getString("key");
if (key != null && key.equals("client.version")) { if (paramKey != null && paramKey.equals("client.version")) {
shortClientVersion = p.getString("value"); shortClientVersion = p.getString("value");
} }
} }
@ -516,9 +530,12 @@ public class YoutubeParsingHelper {
* Get the client version * Get the client version
*/ */
public static String getClientVersion() throws IOException, ExtractionException { public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion; if (!isNullOrEmpty(clientVersion)) {
return clientVersion;
}
if (areHardcodedClientVersionAndKeyValid()) { if (areHardcodedClientVersionAndKeyValid()) {
return clientVersion = HARDCODED_CLIENT_VERSION; clientVersion = HARDCODED_CLIENT_VERSION;
return clientVersion;
} }
extractClientVersionAndKey(); extractClientVersionAndKey();
@ -529,9 +546,12 @@ public class YoutubeParsingHelper {
* Get the key * Get the key
*/ */
public static String getKey() throws IOException, ExtractionException { public static String getKey() throws IOException, ExtractionException {
if (!isNullOrEmpty(key)) return key; if (!isNullOrEmpty(key)) {
return key;
}
if (areHardcodedClientVersionAndKeyValid()) { if (areHardcodedClientVersionAndKeyValid()) {
return key = HARDCODED_KEY; key = HARDCODED_KEY;
return key;
} }
extractClientVersionAndKey(); extractClientVersionAndKey();
@ -574,7 +594,7 @@ public class YoutubeParsingHelper {
+ HARDCODED_YOUTUBE_MUSIC_KEY[0]; + HARDCODED_YOUTUBE_MUSIC_KEY[0];
// @formatter:off // @formatter:off
byte[] json = JsonWriter.string() final byte[] json = JsonWriter.string()
.object() .object()
.object("context") .object("context")
.object("client") .object("client")
@ -617,9 +637,12 @@ public class YoutubeParsingHelper {
public static String[] getYoutubeMusicKey() throws IOException, ReCaptchaException, public static String[] getYoutubeMusicKey() throws IOException, ReCaptchaException,
Parser.RegexException { Parser.RegexException {
if (youtubeMusicKey != null && youtubeMusicKey.length == 3) return youtubeMusicKey; if (youtubeMusicKey != null && youtubeMusicKey.length == 3) {
return youtubeMusicKey;
}
if (isHardcodedYoutubeMusicKeyValid()) { if (isHardcodedYoutubeMusicKeyValid()) {
return youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY; youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY;
return youtubeMusicKey;
} }
final String url = "https://music.youtube.com/"; final String url = "https://music.youtube.com/";
@ -627,31 +650,36 @@ public class YoutubeParsingHelper {
addCookieHeader(headers); addCookieHeader(headers);
final String html = getDownloader().get(url, headers).responseBody(); final String html = getDownloader().get(url, headers).responseBody();
String key; String innertubeApiKey;
try { try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); innertubeApiKey = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (final Parser.RegexException e) { } catch (final Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); innertubeApiKey = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
} }
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", final String innertubeClientName
html); = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
String clientVersion; String innertubeClientVersion;
try { try {
clientVersion = Parser.matchGroup1( innertubeClientVersion = Parser.matchGroup1(
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException e) { } catch (final Parser.RegexException e) {
try { try {
clientVersion = Parser.matchGroup1( innertubeClientVersion = Parser.matchGroup1(
"INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); "INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException ee) { } catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1( innertubeClientVersion = Parser.matchGroup1(
"innertube_context_client_version\":\"([0-9\\.]+?)\"", html); "innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
} }
} }
return youtubeMusicKey = new String[]{key, clientName, clientVersion}; youtubeMusicKey = new String[]{
innertubeApiKey,
innertubeClientName,
innertubeClientVersion
};
return youtubeMusicKey;
} }
@Nullable @Nullable
@ -667,16 +695,14 @@ public class YoutubeParsingHelper {
if (internUrl.startsWith("/redirect?")) { if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter // q parameter can be the first parameter
internUrl = internUrl.substring(10); internUrl = internUrl.substring(10);
String[] params = internUrl.split("&"); final String[] params = internUrl.split("&");
for (String param : params) { for (final String param : params) {
if (param.split("=")[0].equals("q")) { if (param.split("=")[0].equals("q")) {
String url;
try { try {
url = URLDecoder.decode(param.split("=")[1], UTF_8); return URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (final UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
return null; return null;
} }
return url;
} }
} }
} else if (internUrl.startsWith("http")) { } else if (internUrl.startsWith("http")) {
@ -702,7 +728,7 @@ public class YoutubeParsingHelper {
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\""
+ browseEndpoint + "\")"); + browseEndpoint + "\")");
} else if (navigationEndpoint.has("watchEndpoint")) { } else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder(); final StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
.getObject("watchEndpoint").getString("videoId")); .getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
@ -715,8 +741,8 @@ public class YoutubeParsingHelper {
} }
return url.toString(); return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) { } else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" + return "https://www.youtube.com/playlist?list="
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId"); + navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
} }
return null; return null;
} }
@ -731,17 +757,23 @@ public class YoutubeParsingHelper {
@Nullable @Nullable
public static String getTextFromObject(final JsonObject textObject, final boolean html) public static String getTextFromObject(final JsonObject textObject, final boolean html)
throws ParsingException { throws ParsingException {
if (isNullOrEmpty(textObject)) return null; if (isNullOrEmpty(textObject)) {
return null;
}
if (textObject.has("simpleText")) return textObject.getString("simpleText"); if (textObject.has("simpleText")) {
return textObject.getString("simpleText");
}
if (textObject.getArray("runs").isEmpty()) return null; if (textObject.getArray("runs").isEmpty()) {
return null;
}
final StringBuilder textBuilder = new StringBuilder(); final StringBuilder textBuilder = new StringBuilder();
for (final Object textPart : textObject.getArray("runs")) { for (final Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text"); final String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).has("navigationEndpoint")) { if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart) final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
.getObject("navigationEndpoint")); .getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) { if (!isNullOrEmpty(url)) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text) textBuilder.append("<a href=\"").append(url).append("\">").append(text)
@ -768,27 +800,28 @@ public class YoutubeParsingHelper {
} }
@Nullable @Nullable
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String key) public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey)
throws ParsingException { throws ParsingException {
if (jsonObject.isString(key)) { if (jsonObject.isString(theKey)) {
return jsonObject.getString(key); return jsonObject.getString(theKey);
} else { } else {
return getTextFromObject(jsonObject.getObject(key)); return getTextFromObject(jsonObject.getObject(theKey));
} }
} }
public static String fixThumbnailUrl(@Nonnull String thumbnailUrl) { public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) {
if (thumbnailUrl.startsWith("//")) { String result = thumbnailUrl;
thumbnailUrl = thumbnailUrl.substring(2); if (result.startsWith("//")) {
result = result.substring(2);
} }
if (thumbnailUrl.startsWith(HTTP)) { if (result.startsWith(HTTP)) {
thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl); result = Utils.replaceHttpWithHttps(result);
} else if (!thumbnailUrl.startsWith(HTTPS)) { } else if (!result.startsWith(HTTPS)) {
thumbnailUrl = "https://" + thumbnailUrl; result = "https://" + result;
} }
return thumbnailUrl; return result;
} }
public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem) public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem)
@ -882,7 +915,7 @@ public class YoutubeParsingHelper {
public static JsonArray getJsonResponse(final String url, final Localization localization) public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException { throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers); addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization); final Response response = getDownloader().get(url, headers, localization);
@ -1011,7 +1044,8 @@ public class YoutubeParsingHelper {
throws IOException, ExtractionException { throws IOException, ExtractionException {
if (withThirdParty) { if (withThirdParty) {
// @formatter:off // @formatter:off
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId) return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
localization, contentCountry, videoId)
.object("playbackContext") .object("playbackContext")
.object("contentPlaybackContext") .object("contentPlaybackContext")
.value("signatureTimestamp", sts) .value("signatureTimestamp", sts)
@ -1070,7 +1104,7 @@ public class YoutubeParsingHelper {
*/ */
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) { public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) { if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(generateConsentCookie())); headers.put("Cookie", Collections.singletonList(generateConsentCookie()));
} else { } else {
headers.get("Cookie").add(generateConsentCookie()); headers.get("Cookie").add(generateConsentCookie());
} }
@ -1121,15 +1155,27 @@ public class YoutubeParsingHelper {
if (alertText.contains("violation") || alertText.contains("violating") if (alertText.contains("violation") || alertText.contains("violating")
|| alertText.contains("infringement")) { || alertText.contains("infringement")) {
// Possible error messages: // Possible error messages:
// "This account has been terminated for a violation of YouTube's Terms of Service." // "This account has been terminated for a violation of YouTube's Terms of
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech." // Service."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten." // "This account has been terminated due to multiple or severe violations of
// "This account has been terminated due to multiple or severe violations of YouTube's policy against spam, deceptive practices and misleading content or other Terms of Service violations." // YouTube's policy prohibiting hate speech."
// "This account has been terminated due to multiple or severe violations of YouTube's policy on nudity or sexual content." // "This account has been terminated due to multiple or severe violations of
// "This account has been terminated for violating YouTube's Community Guidelines." // YouTube's policy prohibiting content designed to harass, bully or
// "This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted." // threaten."
// "This account has been terminated because it is linked to an account that received multiple third-party claims of copyright infringement." // "This account has been terminated due to multiple or severe violations
throw new AccountTerminatedException(alertText, AccountTerminatedException.Reason.VIOLATION); // of YouTube's policy against spam, deceptive practices and misleading
// content or other Terms of Service violations."
// "This account has been terminated due to multiple or severe violations of
// YouTube's policy on nudity or sexual content."
// "This account has been terminated for violating YouTube's Community
// Guidelines."
// "This account has been terminated because we received multiple
// third-party claims of copyright infringement regarding material that
// the user posted."
// "This account has been terminated because it is linked to an account that
// received multiple third-party claims of copyright infringement."
throw new AccountTerminatedException(alertText,
AccountTerminatedException.Reason.VIOLATION);
} else { } else {
throw new AccountTerminatedException(alertText); throw new AccountTerminatedException(alertText);
} }
@ -1146,8 +1192,8 @@ public class YoutubeParsingHelper {
for (final Object content : contents) { for (final Object content : contents) {
final JsonObject resultObject = (JsonObject) content; final JsonObject resultObject = (JsonObject) content;
if (resultObject.has("itemSectionRenderer")) { if (resultObject.has("itemSectionRenderer")) {
for (final Object sectionContentObject : for (final Object sectionContentObject
resultObject.getObject("itemSectionRenderer").getArray("contents")) { : resultObject.getObject("itemSectionRenderer").getArray("contents")) {
final JsonObject sectionContent = (JsonObject) sectionContentObject; final JsonObject sectionContent = (JsonObject) sectionContentObject;
if (sectionContent.has("infoPanelContentRenderer")) { if (sectionContent.has("infoPanelContentRenderer")) {
@ -1200,8 +1246,8 @@ public class YoutubeParsingHelper {
} }
@Nonnull @Nonnull
private static MetaInfo getClarificationRendererContent(@Nonnull final JsonObject clarificationRenderer) private static MetaInfo getClarificationRendererContent(
throws ParsingException { @Nonnull final JsonObject clarificationRenderer) throws ParsingException {
final MetaInfo metaInfo = new MetaInfo(); final MetaInfo metaInfo = new MetaInfo();
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
@ -1275,7 +1321,7 @@ public class YoutubeParsingHelper {
return false; return false;
} }
for (Object badge : badges) { for (final Object badge : badges) {
final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer") final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer")
.getString("style"); .getString("style");
if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED") if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED")

62
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,11 +1,16 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
@ -42,12 +47,6 @@ import java.util.List;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
/* /*
* Created by Christian Schabesberger on 23.08.15. * Created by Christian Schabesberger on 23.08.15.
* *
@ -70,7 +69,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
public class YoutubeService extends StreamingService { public class YoutubeService extends StreamingService {
public YoutubeService(int id) { public YoutubeService(final int id) {
super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS));
} }
@ -100,12 +99,12 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new YoutubeStreamExtractor(this, linkHandler); return new YoutubeStreamExtractor(this, linkHandler);
} }
@Override @Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new YoutubeChannelExtractor(this, linkHandler); return new YoutubeChannelExtractor(this, linkHandler);
} }
@ -120,7 +119,7 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query) { public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
final List<String> contentFilters = query.getContentFilters(); final List<String> contentFilters = query.getContentFilters();
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) { if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) {
@ -137,22 +136,21 @@ public class YoutubeService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
// add kiosks here e.g.: // add kiosks here e.g.:
try { try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() { list.addKioskEntry(
@Override (streamingService, url, id) -> new YoutubeTrendingExtractor(
public KioskExtractor createNewKiosk(StreamingService streamingService, YoutubeService.this,
String url, new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
String id) id
throws ExtractionException { ),
return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory(),
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); "Trending"
} );
}, new YoutubeTrendingLinkHandlerFactory(), "Trending");
list.setDefaultKiosk("Trending"); list.setDefaultKiosk("Trending");
} catch (Exception e) { } catch (final Exception e) {
throw new ExtractionException(e); throw new ExtractionException(e);
} }
@ -176,7 +174,7 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler)
throws ExtractionException { throws ExtractionException {
return new YoutubeCommentsExtractor(this, urlIdHandler); return new YoutubeCommentsExtractor(this, urlIdHandler);
} }
@ -199,14 +197,14 @@ public class YoutubeService extends StreamingService {
// https://www.youtube.com/picker_ajax?action_country_json=1 // https://www.youtube.com/picker_ajax?action_country_json=1
private static final List<ContentCountry> SUPPORTED_COUNTRIES = ContentCountry.listFrom( private static final List<ContentCountry> SUPPORTED_COUNTRIES = ContentCountry.listFrom(
"DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA", "CL", "DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA",
"CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR", "GE", "DE", "CL", "CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR",
"GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL", "IT", "JM", "JP", "GE", "DE", "GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL",
"JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY", "MT", "MX", "ME", "MA", "IT", "JM", "JP", "JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY",
"NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA", "PG", "PY", "PE", "PH", "PL", "MT", "MX", "ME", "MA", "NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA",
"PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG", "SK", "SI", "ZA", "KR", "ES", "LK", "PG", "PY", "PE", "PH", "PL", "PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG",
"SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG", "UA", "AE", "GB", "US", "UY", "VE", "VN", "SK", "SI", "ZA", "KR", "ES", "LK", "SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG",
"YE", "ZW" "UA", "AE", "GB", "US", "UY", "VE", "VN", "YE", "ZW"
); );
@Override @Override

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View File

@ -80,7 +80,8 @@ public class YoutubeThrottlingDecrypter {
public static String apply(final String url, final String videoId) throws ParsingException { public static String apply(final String url, final String videoId) throws ParsingException {
if (containsNParam(url)) { if (containsNParam(url)) {
if (FUNCTION == null) { if (FUNCTION == null) {
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); final String playerJsCode
= YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
FUNCTION_NAME = parseDecodeFunctionName(playerJsCode); FUNCTION_NAME = parseDecodeFunctionName(playerJsCode);
FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME); FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
@ -118,19 +119,22 @@ public class YoutubeThrottlingDecrypter {
throws Parser.RegexException { throws Parser.RegexException {
try { try {
return parseWithParenthesisMatching(playerJsCode, functionName); return parseWithParenthesisMatching(playerJsCode, functionName);
} catch (Exception e) { } catch (final Exception e) {
return parseWithRegex(playerJsCode, functionName); return parseWithRegex(playerJsCode, functionName);
} }
} }
@Nonnull @Nonnull
private static String parseWithParenthesisMatching(final String playerJsCode, final String functionName) { private static String parseWithParenthesisMatching(final String playerJsCode,
final String functionName) {
final String functionBase = functionName + "=function"; final String functionBase = functionName + "=function";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";"; return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
+ ";";
} }
@Nonnull @Nonnull
private static String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException { private static String parseWithRegex(final String playerJsCode, final String functionName)
throws Parser.RegexException {
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n", final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
Pattern.DOTALL); Pattern.DOTALL);
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
@ -155,7 +159,9 @@ public class YoutubeThrottlingDecrypter {
return Parser.matchGroup1(N_PARAM_PATTERN, url); return Parser.matchGroup1(N_PARAM_PATTERN, url);
} }
private static String decryptNParam(final String function, final String functionName, final String nParam) { private static String decryptNParam(final String function,
final String functionName,
final String nParam) {
if (N_PARAMS_CACHE.containsKey(nParam)) { if (N_PARAMS_CACHE.containsKey(nParam)) {
return N_PARAMS_CACHE.get(nParam); return N_PARAMS_CACHE.get(nParam);
} }

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,5 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
@ -31,11 +43,6 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Created by Christian Schabesberger on 25.07.16. * Created by Christian Schabesberger on 25.07.16.
* *
@ -56,13 +63,12 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor { public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData; private JsonObject initialData;
private JsonObject videoTab; private JsonObject videoTab;
/** /**
* Some channels have response redirects and the only way to reliably get the id is by saving it. * Some channels have response redirects and the only way to reliably get the id is by saving it
* <p> * <p>
* "Movies & Shows": * "Movies & Shows":
* <pre> * <pre>
@ -233,7 +239,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
try { try {
String url = initialData.getObject("header") final String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails") .getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url"); .getObject(0).getString("url");
@ -246,7 +252,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
try { try {
String url = initialData.getObject("header") final String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails") .getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails")
.getObject(0).getString("url"); .getObject(0).getString("url");
@ -361,7 +367,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0) .getObject(0)
.getObject("appendContinuationItemsAction"); .getObject("appendContinuationItemsAction");
@ -436,28 +442,30 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nullable @Nullable
private JsonObject getVideoTab() throws ParsingException { private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab; if (this.videoTab != null) {
return this.videoTab;
}
JsonArray tabs = initialData.getObject("contents") final JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer") .getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs"); .getArray("tabs");
JsonObject videoTab = null;
JsonObject foundVideoTab = null;
for (final Object tab : tabs) { for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) { if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", if (((JsonObject) tab).getObject("tabRenderer").getString("title",
EMPTY_STRING).equals("Videos")) { EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer"); foundVideoTab = ((JsonObject) tab).getObject("tabRenderer");
break; break;
} }
} }
} }
if (videoTab == null) { if (foundVideoTab == null) {
throw new ContentNotSupportedException("This channel has no Videos tab"); throw new ContentNotSupportedException("This channel has no Videos tab");
} }
final String messageRendererText = getTextFromObject(videoTab.getObject("content") final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text")); .getObject("messageRenderer").getObject("text"));
@ -466,7 +474,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return null; return null;
} }
this.videoTab = videoTab; this.videoTab = foundVideoTab;
return videoTab; return foundVideoTab;
} }
} }

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -33,19 +33,20 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
*/ */
public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JsonObject channelInfoItem; private final JsonObject channelInfoItem;
public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) { public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) {
this.channelInfoItem = channelInfoItem; this.channelInfoItem = channelInfoItem;
} }
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); final String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails")
.getObject(0).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
} }
@ -54,7 +55,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return getTextFromObject(channelInfoItem.getObject("title")); return getTextFromObject(channelInfoItem.getObject("title"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
} }
@ -62,9 +63,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
String id = "channel/" + channelInfoItem.getString("channelId"); final String id = "channel/" + channelInfoItem.getString("channelId");
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get url", e); throw new ParsingException("Could not get url", e);
} }
} }
@ -77,8 +78,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
return -1; return -1;
} }
return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText"))); return Utils.mixedNumberWordToLong(getTextFromObject(
} catch (Exception e) { channelInfoItem.getObject("subscriberCountText")));
} catch (final Exception e) {
throw new ParsingException("Could not get subscriber count", e); throw new ParsingException("Could not get subscriber count", e);
} }
} }
@ -93,7 +95,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
channelInfoItem.getObject("videoCountText")))); channelInfoItem.getObject("videoCountText"))));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get stream count", e); throw new ParsingException("Could not get stream count", e);
} }
} }
@ -112,7 +114,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
} }
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get description", e); throw new ParsingException("Could not get description", e);
} }
} }

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -103,7 +103,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
itemSectionRenderer itemSectionRenderer
.getObject("itemSectionRenderer") .getObject("itemSectionRenderer")
.getArray("contents").getObject(0), .getArray("contents").getObject(0),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token"); "continuationItemRenderer.continuationEndpoint"
+ ".continuationCommand.token");
} catch (final ParsingException ignored) { } catch (final ParsingException ignored) {
return null; return null;
} }

40
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -1,9 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -14,9 +16,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json; private final JsonObject json;
@ -33,11 +32,12 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
private JsonObject getCommentRenderer() throws ParsingException { private JsonObject getCommentRenderer() throws ParsingException {
if(commentRenderer == null) { if (commentRenderer == null) {
if(!json.has("comment")) if (json.has("comment")) {
commentRenderer = json;
else
commentRenderer = JsonUtils.getObject(json, "comment.commentRenderer"); commentRenderer = JsonUtils.getObject(json, "comment.commentRenderer");
} else {
commentRenderer = json;
}
} }
return commentRenderer; return commentRenderer;
} }
@ -50,7 +50,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails"); final JsonArray arr = JsonUtils.getArray(getCommentRenderer(),
"authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url"); return JsonUtils.getString(arr.getObject(2), "url");
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
@ -69,7 +70,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getTextualUploadDate() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
try { try {
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "publishedTimeText")); return getTextFromObject(JsonUtils.getObject(getCommentRenderer(),
"publishedTimeText"));
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get publishedTimeText", e); throw new ParsingException("Could not get publishedTimeText", e);
} }
@ -78,7 +80,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Nullable @Nullable
@Override @Override
public DateWrapper getUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
String textualPublishedTime = getTextualUploadDate(); final String textualPublishedTime = getTextualUploadDate();
if (timeAgoParser != null && textualPublishedTime != null if (timeAgoParser != null && textualPublishedTime != null
&& !textualPublishedTime.isEmpty()) { && !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime); return timeAgoParser.parse(textualPublishedTime);
@ -108,7 +110,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
final String likeCount; final String likeCount;
try { try {
likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(getCommentRenderer(), likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(getCommentRenderer(),
"actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer.accessibilityData.accessibilityData.label")); "actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer"
+ ".accessibilityData.accessibilityData.label"));
} catch (final Exception e) { } catch (final Exception e) {
// Use the approximate like count returned into the voteCount object // Use the approximate like count returned into the voteCount object
// This may return a language dependent version, e.g. in German: 3,3 Mio // This may return a language dependent version, e.g. in German: 3,3 Mio
@ -202,7 +205,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails"); final JsonArray arr = JsonUtils.getArray(getCommentRenderer(),
"authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url"); return JsonUtils.getString(arr.getObject(2), "url");
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get author thumbnail", e); throw new ParsingException("Could not get author thumbnail", e);
@ -211,7 +215,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public boolean isHeartedByUploader() throws ParsingException { public boolean isHeartedByUploader() throws ParsingException {
final JsonObject commentActionButtonsRenderer = getCommentRenderer().getObject("actionButtons") final JsonObject commentActionButtonsRenderer = getCommentRenderer()
.getObject("actionButtons")
.getObject("commentActionButtonsRenderer"); .getObject("commentActionButtonsRenderer");
return commentActionButtonsRenderer.has("creatorHeart"); return commentActionButtonsRenderer.has("creatorHeart");
} }
@ -247,10 +252,13 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public Page getReplies() throws ParsingException { public Page getReplies() throws ParsingException {
try { try {
final String id = JsonUtils.getString(JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents").getObject(0), "continuationItemRenderer.continuationEndpoint.continuationCommand.token"); final String id = JsonUtils.getString(
JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents")
.getObject(0),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
return new Page(url, id); return new Page(url, id);
} catch (final Exception e) { } catch (final Exception e) {
return null; // Would return null for Comment Replies, since YouTube does not support nested replies. return null;
} }
} }
} }

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java
View File

@ -22,14 +22,15 @@ import java.io.IOException;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
public class YoutubeFeedExtractor extends FeedExtractor { public class YoutubeFeedExtractor extends FeedExtractor {
public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) { public YoutubeFeedExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
private Document document; private Document document;
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelIdOrUser = getLinkHandler().getId(); final String channelIdOrUser = getLinkHandler().getId();
final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser); final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
@ -46,7 +47,7 @@ public class YoutubeFeedExtractor extends FeedExtractor {
final Elements entries = document.select("feed > entry"); final Elements entries = document.select("feed > entry");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
for (Element entryElement : entries) { for (final Element entryElement : entries) {
collector.commit(new YoutubeFeedInfoItemExtractor(entryElement)); collector.commit(new YoutubeFeedInfoItemExtractor(entryElement));
} }

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java
View File

@ -13,7 +13,7 @@ import java.time.format.DateTimeParseException;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement; private final Element entryElement;
public YoutubeFeedInfoItemExtractor(Element entryElement) { public YoutubeFeedInfoItemExtractor(final Element entryElement) {
this.entryElement = entryElement; this.entryElement = entryElement;
} }
@ -37,7 +37,8 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public long getViewCount() { public long getViewCount() {
return Long.parseLong(entryElement.getElementsByTag("media:statistics").first().attr("views")); return Long.parseLong(entryElement.getElementsByTag("media:statistics").first()
.attr("views"));
} }
@Override @Override
@ -72,8 +73,9 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
public DateWrapper getUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
try { try {
return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate())); return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate()));
} catch (DateTimeParseException e) { } catch (final DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e); throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")",
e);
} }
} }

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -1,10 +1,21 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
@ -25,14 +36,14 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
/** /**
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
* It handles URLs in the format of * It handles URLs in the format of
@ -84,8 +95,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist"); .getObject("playlist").getObject("playlist");
if (isNullOrEmpty(playlistData)) throw new ExtractionException( if (isNullOrEmpty(playlistData)) {
"Could not get playlistData"); throw new ExtractionException("Could not get playlistData");
}
cookieValue = extractCookieValue(COOKIE_NAME, response); cookieValue = extractCookieValue(COOKIE_NAME, response);
} }

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -1,6 +1,24 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
@ -19,17 +37,14 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor { public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData; private JsonObject initialData;
@ -162,7 +177,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return false; return false;
} }
JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0); final JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
return firstContent.has("didYouMeanRenderer") return firstContent.has("didYouMeanRenderer")
|| firstContent.has("showingResultsForRenderer"); || firstContent.has("showingResultsForRenderer");
@ -211,7 +226,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
// @formatter:off // @formatter:off
byte[] json = JsonWriter.string() final byte[] json = JsonWriter.string()
.object() .object()
.object("context") .object("context")
.object("client") .object("client")
@ -331,7 +346,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
if (searchType.equals(MUSIC_VIDEOS)) { if (searchType.equals(MUSIC_VIDEOS)) {
JsonArray items = info.getObject("menu").getObject("menuRenderer") final JsonArray items = info.getObject("menu")
.getObject("menuRenderer")
.getArray("items"); .getArray("items");
for (final Object item : items) { for (final Object item : items) {
final JsonObject menuNavigationItemRenderer = final JsonObject menuNavigationItemRenderer =
@ -354,8 +370,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0); .getObject("text").getArray("runs").getObject(0);
if (!navigationEndpointHolder.has("navigationEndpoint")) if (!navigationEndpointHolder.has("navigationEndpoint")) {
return null; return null;
}
final String url = getUrlFromNavigationEndpoint( final String url = getUrlFromNavigationEndpoint(
navigationEndpointHolder.getObject("navigationEndpoint")); navigationEndpointHolder.getObject("navigationEndpoint"));

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -1,5 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
@ -30,9 +42,6 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
// Minimum size of the stats array in the browse response which includes the streams count // Minimum size of the stats array in the browse response which includes the streams count
private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2; private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2;

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -11,20 +11,20 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private JsonObject playlistInfoItem; private final JsonObject playlistInfoItem;
public YoutubePlaylistInfoItemExtractor(JsonObject playlistInfoItem) { public YoutubePlaylistInfoItemExtractor(final JsonObject playlistInfoItem) {
this.playlistInfoItem = playlistInfoItem; this.playlistInfoItem = playlistInfoItem;
} }
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
String url = playlistInfoItem.getArray("thumbnails").getObject(0) final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url"); .getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
} }
@ -33,7 +33,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return getTextFromObject(playlistInfoItem.getObject("title")); return getTextFromObject(playlistInfoItem.getObject("title"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
} }
@ -41,9 +41,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
String id = playlistInfoItem.getString("playlistId"); final String id = playlistInfoItem.getString("playlistId");
return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id); return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get url", e); throw new ParsingException("Could not get url", e);
} }
} }
@ -52,7 +52,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return getTextFromObject(playlistInfoItem.getObject("longBylineText")); return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get uploader name", e); throw new ParsingException("Could not get uploader name", e);
} }
} }
@ -60,8 +60,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
return Long.parseLong(Utils.removeNonDigitCharacters(playlistInfoItem.getString("videoCount"))); return Long.parseLong(Utils.removeNonDigitCharacters(
} catch (Exception e) { playlistInfoItem.getString("videoCount")));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e); throw new ParsingException("Could not get stream count", e);
} }
} }

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -1,6 +1,22 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
@ -16,15 +32,11 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Created by Christian Schabesberger on 22.07.2018 * Created by Christian Schabesberger on 22.07.2018
@ -179,7 +191,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final JsonObject ajaxJson; final JsonObject ajaxJson;
try { try {
ajaxJson = JsonParser.object().from(responseBody); ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e); throw new ParsingException("Could not parse JSON", e);
} }

149
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1,5 +1,18 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createPlayerBodyWithSts;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonMobilePostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileEmbedVideoJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopEmbedVideoJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
@ -7,7 +20,6 @@ import com.grack.nanojson.JsonWriter;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
@ -32,24 +44,35 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Created by Christian Schabesberger on 06.08.15. * Created by Christian Schabesberger on 06.08.15.
@ -127,7 +150,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (isNullOrEmpty(title)) { if (isNullOrEmpty(title)) {
title = playerResponse.getObject("videoDetails").getString("title"); title = playerResponse.getObject("videoDetails").getString("title");
if (isNullOrEmpty(title)) throw new ParsingException("Could not get name"); if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get name");
}
} }
return title; return title;
@ -158,8 +183,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.startsWith("Premiered")) { .startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) final String time = getTextFromObject(
.substring(10); getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago try { // Premiered 20 hours ago
final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor( final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
@ -206,10 +231,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
try { try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail") final JsonArray thumbnails = playerResponse.getObject("videoDetails")
.getArray("thumbnails"); .getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (final Exception e) { } catch (final Exception e) {
@ -224,9 +249,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
// Description with more info on links // Description with more info on links
try { try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer() final String description = getTextFromObject(getVideoSecondaryInfoRenderer()
.getObject("description"), true); .getObject("description"), true);
if (!isNullOrEmpty(description)) return new Description(description, Description.HTML); if (!isNullOrEmpty(description)) {
return new Description(description, Description.HTML);
}
} catch (final ParsingException ignored) { } catch (final ParsingException ignored) {
// Age-restricted videos cause a ParsingException here // Age-restricted videos cause a ParsingException here
} }
@ -327,10 +354,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (isNullOrEmpty(views)) { if (isNullOrEmpty(views)) {
views = playerResponse.getObject("videoDetails").getString("viewCount"); views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count"); if (isNullOrEmpty(views)) {
throw new ParsingException("Could not get view count");
}
} }
if (views.toLowerCase().contains("no views")) return 0; if (views.toLowerCase().contains("no views")) {
return 0;
}
return Long.parseLong(Utils.removeNonDigitCharacters(views)); return Long.parseLong(Utils.removeNonDigitCharacters(views));
} }
@ -354,7 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (likesString == null) { if (likesString == null) {
// If this kicks in our button has no content and therefore ratings must be disabled // If this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the like button is missing"); throw new ParsingException(
"Ratings are enabled even though the like button is missing");
} }
return -1; return -1;
} }
@ -400,7 +432,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// The difference between the real name of the channel and the displayed name is especially // The difference between the real name of the channel and the displayed name is especially
// visible for music channels and autogenerated channels. // visible for music channels and autogenerated channels.
final String uploaderName = playerResponse.getObject("videoDetails").getString("author"); final String uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name"); if (isNullOrEmpty(uploaderName)) {
throw new ParsingException("Could not get uploader name");
}
return uploaderName; return uploaderName;
} }
@ -440,12 +474,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public long getUploaderSubscriberCount() throws ParsingException { public long getUploaderSubscriberCount() throws ParsingException {
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer"); final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer,
"owner.videoOwnerRenderer");
if (!videoOwnerRenderer.has("subscriberCountText")) { if (!videoOwnerRenderer.has("subscriberCountText")) {
return UNKNOWN_SUBSCRIBER_COUNT; return UNKNOWN_SUBSCRIBER_COUNT;
} }
try { try {
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText"))); return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer
.getObject("subscriberCountText")));
} catch (final NumberFormatException e) { } catch (final NumberFormatException e) {
throw new ParsingException("Could not get uploader subscriber count", e); throw new ParsingException("Could not get uploader subscriber count", e);
} }
@ -677,7 +713,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate"; private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
private static final String[] REGEXES = { private static final String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
+ "\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
"\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)", "\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)",
"\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)", "\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)",
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;",
@ -720,7 +757,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING) final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
.contains("age"); .contains("age");
if (!playerResponse.has("streamingData")) { if (!playerResponse.has("streamingData")) {
@ -765,19 +802,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
@Nonnull JsonObject playabilityStatus) @Nonnull final JsonObject playabilityStatus)
throws ParsingException { throws ParsingException {
String status = playabilityStatus.getString("status"); String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw the specific exception based on error message // If status exist, and is not "OK", throw the specific exception based on error message
// or a ContentNotAvailableException with the reason text if it's an unknown reason. // or a ContentNotAvailableException with the reason text if it's an unknown reason.
if (status != null && !status.equalsIgnoreCase("ok")) { if (status != null && !status.equalsIgnoreCase("ok")) {
playabilityStatus = youtubePlayerResponse.getObject("playabilityStatus"); final JsonObject newPlayabilityStatus
status = playabilityStatus.getString("status"); = youtubePlayerResponse.getObject("playabilityStatus");
final String reason = playabilityStatus.getString("reason"); status = newPlayabilityStatus.getString("status");
final String reason = newPlayabilityStatus.getString("reason");
if (status.equalsIgnoreCase("login_required")) { if (status.equalsIgnoreCase("login_required")) {
if (reason == null) { if (reason == null) {
final String message = playabilityStatus.getArray("messages").getString(0); final String message = newPlayabilityStatus.getArray("messages").getString(0);
if (message != null && message.contains("private")) { if (message != null && message.contains("private")) {
throw new PrivateContentException("This video is private."); throw new PrivateContentException("This video is private.");
} }
@ -797,11 +835,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PaidContentException("This video is a paid video"); throw new PaidContentException("This video is a paid video");
} }
if (reason.contains("members-only")) { if (reason.contains("members-only")) {
throw new PaidContentException( throw new PaidContentException("This video is only available"
"This video is only available for members of the channel of this video"); + " for members of the channel of this video");
} }
if (reason.contains("unavailable")) { if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(playabilityStatus final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen").getObject("playerErrorMessageRenderer") .getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("subreason")); .getObject("subreason"));
if (detailedErrorMessage != null) { if (detailedErrorMessage != null) {
@ -900,8 +938,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String videoId) final String videoId)
throws IOException, ExtractionException { throws IOException, ExtractionException {
final byte[] androidMobileEmbedBody = JsonWriter.string( final byte[] androidMobileEmbedBody = JsonWriter.string(
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId) prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player", final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
androidMobileEmbedBody, contentCountry, localization); androidMobileEmbedBody, contentCountry, localization);
@ -953,11 +991,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return false; return false;
} }
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { private String getDeobfuscationFuncName(final String thePlayerCode)
throws DeobfuscateException {
Parser.RegexException exception = null; Parser.RegexException exception = null;
for (final String regex : REGEXES) { for (final String regex : REGEXES) {
try { try {
return Parser.matchGroup1(regex, playerCode); return Parser.matchGroup1(regex, thePlayerCode);
} catch (final Parser.RegexException re) { } catch (final Parser.RegexException re) {
if (exception == null) { if (exception == null) {
exception = re; exception = re;
@ -1011,7 +1050,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
private void getStsFromPlayerJs() throws ParsingException { private void getStsFromPlayerJs() throws ParsingException {
if (!isNullOrEmpty(sts)) return; if (!isNullOrEmpty(sts)) {
return;
}
if (playerCode == null) { if (playerCode == null) {
storePlayerJs(); storePlayerJs();
if (playerCode == null) { if (playerCode == null) {
@ -1045,51 +1086,55 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; if (this.videoPrimaryInfoRenderer != null) {
return this.videoPrimaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents") final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results") .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents"); .getArray("contents");
JsonObject videoPrimaryInfoRenderer = null; JsonObject theVideoPrimaryInfoRenderer = null;
for (final Object content : contents) { for (final Object content : contents) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) { if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content) theVideoPrimaryInfoRenderer = ((JsonObject) content)
.getObject("videoPrimaryInfoRenderer"); .getObject("videoPrimaryInfoRenderer");
break; break;
} }
} }
if (isNullOrEmpty(videoPrimaryInfoRenderer)) { if (isNullOrEmpty(theVideoPrimaryInfoRenderer)) {
throw new ParsingException("Could not find videoPrimaryInfoRenderer"); throw new ParsingException("Could not find videoPrimaryInfoRenderer");
} }
this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer; this.videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
return videoPrimaryInfoRenderer; return theVideoPrimaryInfoRenderer;
} }
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; if (this.videoSecondaryInfoRenderer != null) {
return this.videoSecondaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents") final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results") .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents"); .getArray("contents");
JsonObject videoSecondaryInfoRenderer = null; JsonObject theVideoSecondaryInfoRenderer = null;
for (final Object content : contents) { for (final Object content : contents) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content) theVideoSecondaryInfoRenderer = ((JsonObject) content)
.getObject("videoSecondaryInfoRenderer"); .getObject("videoSecondaryInfoRenderer");
break; break;
} }
} }
if (isNullOrEmpty(videoSecondaryInfoRenderer)) { if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) {
throw new ParsingException("Could not find videoSecondaryInfoRenderer"); throw new ParsingException("Could not find videoSecondaryInfoRenderer");
} }
this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer; this.videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
return videoSecondaryInfoRenderer; return theVideoSecondaryInfoRenderer;
} }
@Nonnull @Nonnull
@ -1120,8 +1165,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (streamingData != null && streamingData.has(streamingDataKey)) { if (streamingData != null && streamingData.has(streamingDataKey)) {
final JsonArray formats = streamingData.getArray(streamingDataKey); final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) { for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i); final JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag"); final int itag = formatData.getInt("itag");
if (ItagItem.isSupported(itag)) { if (ItagItem.isSupported(itag)) {
try { try {

89
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -1,7 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -12,15 +19,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.time.Instant; import java.time.Instant;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -51,7 +55,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
* @param videoInfoItem The JSON page element * @param videoInfoItem The JSON page element
* @param timeAgoParser A parser of the textual dates or {@code null}. * @param timeAgoParser A parser of the textual dates or {@code null}.
*/ */
public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) { public YoutubeStreamInfoItemExtractor(final JsonObject videoInfoItem,
@Nullable final TimeAgoParser timeAgoParser) {
this.videoInfo = videoInfoItem; this.videoInfo = videoInfoItem;
this.timeAgoParser = timeAgoParser; this.timeAgoParser = timeAgoParser;
} }
@ -64,43 +69,51 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
final JsonArray badges = videoInfo.getArray("badges"); final JsonArray badges = videoInfo.getArray("badges");
for (final Object badge : badges) { for (final Object badge : badges) {
final JsonObject badgeRenderer = ((JsonObject) badge).getObject("metadataBadgeRenderer"); final JsonObject badgeRenderer
if (badgeRenderer.getString("style", EMPTY_STRING).equals("BADGE_STYLE_TYPE_LIVE_NOW") || = ((JsonObject) badge).getObject("metadataBadgeRenderer");
badgeRenderer.getString("label", EMPTY_STRING).equals("LIVE NOW")) { if (badgeRenderer.getString("style", EMPTY_STRING).equals("BADGE_STYLE_TYPE_LIVE_NOW")
return cachedStreamType = StreamType.LIVE_STREAM; || badgeRenderer.getString("label", EMPTY_STRING).equals("LIVE NOW")) {
cachedStreamType = StreamType.LIVE_STREAM;
return cachedStreamType;
} }
} }
for (final Object overlay : videoInfo.getArray("thumbnailOverlays")) { for (final Object overlay : videoInfo.getArray("thumbnailOverlays")) {
final String style = ((JsonObject) overlay) final String style = ((JsonObject) overlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING); .getObject("thumbnailOverlayTimeStatusRenderer")
.getString("style", EMPTY_STRING);
if (style.equalsIgnoreCase("LIVE")) { if (style.equalsIgnoreCase("LIVE")) {
return cachedStreamType = StreamType.LIVE_STREAM; cachedStreamType = StreamType.LIVE_STREAM;
return cachedStreamType;
} }
} }
return cachedStreamType = StreamType.VIDEO_STREAM; cachedStreamType = StreamType.VIDEO_STREAM;
return cachedStreamType;
} }
@Override @Override
public boolean isAd() throws ParsingException { public boolean isAd() throws ParsingException {
return isPremium() || getName().equals("[Private video]") || getName().equals("[Deleted video]"); return isPremium() || getName().equals("[Private video]")
|| getName().equals("[Deleted video]");
} }
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
String videoId = videoInfo.getString("videoId"); final String videoId = videoInfo.getString("videoId");
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get url", e); throw new ParsingException("Could not get url", e);
} }
} }
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
String name = getTextFromObject(videoInfo.getObject("title")); final String name = getTextFromObject(videoInfo.getObject("title"));
if (!isNullOrEmpty(name)) return name; if (!isNullOrEmpty(name)) {
return name;
}
throw new ParsingException("Could not get name"); throw new ParsingException("Could not get name");
} }
@ -113,14 +126,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
String duration = getTextFromObject(videoInfo.getObject("lengthText")); String duration = getTextFromObject(videoInfo.getObject("lengthText"));
if (isNullOrEmpty(duration)) { if (isNullOrEmpty(duration)) {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { for (final Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) { if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay) duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
} }
} }
if (isNullOrEmpty(duration)) throw new ParsingException("Could not get duration"); if (isNullOrEmpty(duration)) {
throw new ParsingException("Could not get duration");
}
} }
return YoutubeParsingHelper.parseDurationString(duration); return YoutubeParsingHelper.parseDurationString(duration);
@ -136,7 +151,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (isNullOrEmpty(name)) { if (isNullOrEmpty(name)) {
name = getTextFromObject(videoInfo.getObject("shortBylineText")); name = getTextFromObject(videoInfo.getObject("shortBylineText"));
if (isNullOrEmpty(name)) throw new ParsingException("Could not get uploader name"); if (isNullOrEmpty(name)) {
throw new ParsingException("Could not get uploader name");
}
} }
} }
@ -156,7 +173,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint")); .getArray("runs").getObject(0).getObject("navigationEndpoint"));
if (isNullOrEmpty(url)) throw new ParsingException("Could not get uploader url"); if (isNullOrEmpty(url)) {
throw new ParsingException("Could not get uploader url");
}
} }
} }
@ -168,7 +187,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
if (videoInfo.has("channelThumbnailSupportedRenderers")) { if (videoInfo.has("channelThumbnailSupportedRenderers")) {
return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails") return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers"
+ ".channelThumbnailWithLinkRenderer.thumbnail.thumbnails")
.getObject(0).getString("url"); .getObject(0).getString("url");
} }
@ -196,8 +216,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere()); return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
} }
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); final String publishedTimeText
if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText; = getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (publishedTimeText != null && !publishedTimeText.isEmpty()) {
return publishedTimeText;
}
return null; return null;
} }
@ -217,7 +240,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (timeAgoParser != null && !isNullOrEmpty(textualUploadDate)) { if (timeAgoParser != null && !isNullOrEmpty(textualUploadDate)) {
try { try {
return timeAgoParser.parse(textualUploadDate); return timeAgoParser.parse(textualUploadDate);
} catch (ParsingException e) { } catch (final ParsingException e) {
throw new ParsingException("Could not get upload date", e); throw new ParsingException("Could not get upload date", e);
} }
} }
@ -245,7 +268,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
} }
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get view count", e); throw new ParsingException("Could not get view count", e);
} }
} }
@ -256,9 +279,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
} }
private boolean isPremium() { private boolean isPremium() {
JsonArray badges = videoInfo.getArray("badges"); final JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) { for (final Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) { if (((JsonObject) badge).getObject("metadataBadgeRenderer")
.getString("label", EMPTY_STRING).equals("Premium")) {
return true; return true;
} }
} }
@ -276,8 +300,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try { try {
return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)), return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
ZoneOffset.UTC); ZoneOffset.UTC);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
} }
} }
@ -286,7 +310,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getShortDescription() throws ParsingException { public String getShortDescription() throws ParsingException {
if (videoInfo.has("detailedMetadataSnippets")) { if (videoInfo.has("detailedMetadataSnippets")) {
return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText")); return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets")
.getObject(0).getObject("snippetText"));
} }
if (videoInfo.has("descriptionSnippet")) { if (videoInfo.has("descriptionSnippet")) {

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View File

@ -70,7 +70,7 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
final JsonArray subscriptions; final JsonArray subscriptions;
try { try {
subscriptions = JsonParser.array().from(contentInputStream); subscriptions = JsonParser.array().from(contentInputStream);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new InvalidSourceException("Invalid json input stream", e); throw new InvalidSourceException("Invalid json input stream", e);
} }
@ -101,7 +101,7 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
public List<SubscriptionItem> fromZipInputStream(@Nonnull final InputStream contentInputStream) public List<SubscriptionItem> fromZipInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException { throws ExtractionException {
try (final ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) { try (ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) {
ZipEntry zipEntry; ZipEntry zipEntry;
while ((zipEntry = zipInputStream.getNextEntry()) != null) { while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (zipEntry.getName().toLowerCase().endsWith(".csv")) { if (zipEntry.getName().toLowerCase().endsWith(".csv")) {
@ -122,15 +122,16 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
throw new InvalidSourceException("Error reading contents of zip file", e); throw new InvalidSourceException("Error reading contents of zip file", e);
} }
throw new InvalidSourceException("Unable to find a valid subscriptions.csv file (try extracting and selecting the csv file)"); throw new InvalidSourceException("Unable to find a valid subscriptions.csv file"
+ " (try extracting and selecting the csv file)");
} }
public List<SubscriptionItem> fromCsvInputStream(@Nonnull final InputStream contentInputStream) public List<SubscriptionItem> fromCsvInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException { throws ExtractionException {
// Expected format of CSV file: // Expected format of CSV file:
// Channel Id,Channel Url,Channel Title // Channel Id,Channel Url,Channel Title
// UC1JTQBa5QxZCpXrFSkMxmPw,http://www.youtube.com/channel/UC1JTQBa5QxZCpXrFSkMxmPw,Raycevick //UC1JTQBa5QxZCpXrFSkMxmPw,http://www.youtube.com/channel/UC1JTQBa5QxZCpXrFSkMxmPw,Raycevick
// UCFl7yKfcRcFmIUbKeCA-SJQ,http://www.youtube.com/channel/UCFl7yKfcRcFmIUbKeCA-SJQ,Joji //UCFl7yKfcRcFmIUbKeCA-SJQ,http://www.youtube.com/channel/UCFl7yKfcRcFmIUbKeCA-SJQ,Joji
// //
// Notes: // Notes:
// It's always 3 columns // It's always 3 columns

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -1,8 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
@ -12,10 +16,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import java.util.Map;
/* /*
* Created by Christian Schabesberger on 28.09.16. * Created by Christian Schabesberger on 28.09.16.
@ -39,12 +43,12 @@ import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSuggestionExtractor extends SuggestionExtractor { public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public YoutubeSuggestionExtractor(StreamingService service) { public YoutubeSuggestionExtractor(final StreamingService service) {
super(service); super(service);
} }
@Override @Override
public List<String> suggestionList(String query) throws IOException, ExtractionException { public List<String> suggestionList(final String query) throws IOException, ExtractionException {
final Downloader dl = NewPipe.getDownloader(); final Downloader dl = NewPipe.getDownloader();
final List<String> suggestions = new ArrayList<>(); final List<String> suggestions = new ArrayList<>();
@ -62,16 +66,20 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
// trim JSONP part "JP(...)" // trim JSONP part "JP(...)"
response = response.substring(3, response.length() - 1); response = response.substring(3, response.length() - 1);
try { try {
JsonArray collection = JsonParser.array().from(response).getArray(1); final JsonArray collection = JsonParser.array().from(response).getArray(1);
for (Object suggestion : collection) { for (final Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) continue; if (!(suggestion instanceof JsonArray)) {
String suggestionStr = ((JsonArray) suggestion).getString(0); continue;
if (suggestionStr == null) continue; }
final String suggestionStr = ((JsonArray) suggestion).getString(0);
if (suggestionStr == null) {
continue;
}
suggestions.add(suggestionStr); suggestions.add(suggestionStr);
} }
return suggestions; return suggestions;
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
} }

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -55,7 +55,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
// @formatter:off // @formatter:off
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry()) getExtractorContentCountry())
@ -92,15 +93,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonArray itemSectionRenderers = initialData.getObject("contents") final JsonArray itemSectionRenderers = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0) .getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") .getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents"); .getArray("contents");
for (final Object itemSectionRenderer : itemSectionRenderers) { for (final Object itemSectionRenderer : itemSectionRenderers) {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer) final JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer)
.getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("shelfRenderer").getObject("content") .getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer"); .getObject("expandedShelfContentsRenderer");

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View File

@ -29,30 +29,34 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory(); private static final YoutubeChannelLinkHandlerFactory INSTANCE
= new YoutubeChannelLinkHandlerFactory();
private static final Pattern excludedSegments = private static final Pattern EXCLUDED_SEGMENTS =
Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site");
private YoutubeChannelLinkHandlerFactory() {
}
public static YoutubeChannelLinkHandlerFactory getInstance() { public static YoutubeChannelLinkHandlerFactory getInstance() {
return instance; return INSTANCE;
} }
/** /**
* Returns URL to channel from an ID * Returns URL to channel from an ID
* *
* @param id Channel ID including e.g. 'channel/' * @param id Channel ID including e.g. 'channel/'
* @param contentFilters
* @param searchFilter
* @return URL to channel * @return URL to channel
*/ */
@Override @Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) { public String getUrl(final String id,
final List<String> contentFilters,
final String searchFilter) {
return "https://www.youtube.com/" + id; return "https://www.youtube.com/" + id;
} }
/** /**
* Returns true if path conform to * Returns true if path conform to
* custom short channel URLs like youtube.com/yourcustomname * custom short channel URLs like youtube.com/yourcustomname
@ -61,17 +65,18 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
* @return true - if value conform to short channel URL, false - not * @return true - if value conform to short channel URL, false - not
*/ */
private boolean isCustomShortChannelUrl(final String[] splitPath) { private boolean isCustomShortChannelUrl(final String[] splitPath) {
return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches(); return splitPath.length == 1 && !EXCLUDED_SEGMENTS.matcher(splitPath[0]).matches();
} }
@Override @Override
public String getId(String url) throws ParsingException { public String getId(final String url) throws ParsingException {
try { try {
final URL urlObj = Utils.stringToURL(url); final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath(); String path = urlObj.getPath();
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) { || YoutubeParsingHelper.isInvidioURL(urlObj)
|| YoutubeParsingHelper.isHooktubeURL(urlObj))) {
throw new ParsingException("the URL given is not a Youtube-URL"); throw new ParsingException("the URL given is not a Youtube-URL");
} }
@ -85,7 +90,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
splitPath = path.split("/"); splitPath = path.split("/");
} }
if (!path.startsWith("user/") && !path.startsWith("channel/") && !path.startsWith("c/")) { if (!path.startsWith("user/")
&& !path.startsWith("channel/")
&& !path.startsWith("c/")) {
throw new ParsingException("the URL given is neither a channel nor an user"); throw new ParsingException("the URL given is neither a channel nor an user");
} }
@ -97,15 +104,16 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
return splitPath[0] + "/" + id; return splitPath[0] + "/" + id;
} catch (final Exception exception) { } catch (final Exception exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); throw new ParsingException("Error could not parse url :" + exception.getMessage(),
exception);
} }
} }
@Override @Override
public boolean onAcceptUrl(String url) { public boolean onAcceptUrl(final String url) {
try { try {
getId(url); getId(url);
} catch (ParsingException e) { } catch (final ParsingException e) {
return false; return false;
} }
return true; return true;

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View File

@ -6,22 +6,27 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List; import java.util.List;
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeCommentsLinkHandlerFactory instance = new YoutubeCommentsLinkHandlerFactory(); private static final YoutubeCommentsLinkHandlerFactory INSTANCE
= new YoutubeCommentsLinkHandlerFactory();
private YoutubeCommentsLinkHandlerFactory() {
}
public static YoutubeCommentsLinkHandlerFactory getInstance() { public static YoutubeCommentsLinkHandlerFactory getInstance() {
return instance; return INSTANCE;
} }
@Override @Override
public String getUrl(String id) { public String getUrl(final String id) {
return "https://www.youtube.com/watch?v=" + id; return "https://www.youtube.com/watch?v=" + id;
} }
@Override @Override
public String getId(String urlString) throws ParsingException, IllegalArgumentException { public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); //we need the same id, avoids duplicate code // we need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
} }
@Override @Override
@ -29,15 +34,17 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
try { try {
getId(url); getId(url);
return true; return true;
} catch (FoundAdException fe) { } catch (final FoundAdException fe) {
throw fe; throw fe;
} catch (ParsingException e) { } catch (final ParsingException e) {
return false; return false;
} }
} }
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return getUrl(id); return getUrl(id);
} }
} }

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -11,11 +11,14 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubePlaylistLinkHandlerFactory INSTANCE = private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
new YoutubePlaylistLinkHandlerFactory(); new YoutubePlaylistLinkHandlerFactory();
private YoutubePlaylistLinkHandlerFactory() {
}
public static YoutubePlaylistLinkHandlerFactory getInstance() { public static YoutubePlaylistLinkHandlerFactory getInstance() {
return INSTANCE; return INSTANCE;
} }
@ -54,8 +57,10 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
&& Utils.getQueryValue(urlObj, "v") == null) { && Utils.getQueryValue(urlObj, "v") == null) {
//Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId // Video id can't be determined from the channel mix id.
throw new ContentNotSupportedException("Channel Mix without a video id are not supported"); // See YoutubeParsingHelper#extractVideoIdFromMixId
throw new ContentNotSupportedException(
"Channel Mix without a video id are not supported");
} }
return listID; return listID;
@ -69,15 +74,15 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
public boolean onAcceptUrl(final String url) { public boolean onAcceptUrl(final String url) {
try { try {
getId(url); getId(url);
} catch (ParsingException e) { } catch (final ParsingException e) {
return false; return false;
} }
return true; return true;
} }
/** /**
* If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is like * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is
* {@code https://youtube.com/watch?v=videoId&list=playlistId} * like {@code https://youtube.com/watch?v=videoId&list=playlistId}
* <p>Otherwise use super</p> * <p>Otherwise use super</p>
*/ */
@Override @Override
@ -96,7 +101,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
getContentFilter(url), getContentFilter(url),
getSortFilter(url)); getSortFilter(url));
} }
} catch (MalformedURLException exception) { } catch (final MalformedURLException exception) {
throw new ParsingException("Error could not parse URL: " + exception.getMessage(), throw new ParsingException("Error could not parse URL: " + exception.getMessage(),
exception); exception);
} }

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -11,7 +11,7 @@ import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all"; public static final String ALL = "all";
public static final String VIDEOS = "videos"; public static final String VIDEOS = "videos";
@ -84,7 +84,10 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Nonnull @Nonnull
public static String getSearchParameter(final String contentFilter) { public static String getSearchParameter(final String contentFilter) {
if (isNullOrEmpty(contentFilter)) return ""; if (isNullOrEmpty(contentFilter)) {
return "";
}
switch (contentFilter) { switch (contentFilter) {
case VIDEOS: case VIDEOS:
return "EgIQAQ%3D%3D"; return "EgIQAQ%3D%3D";

97
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View File

@ -1,12 +1,16 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler; package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
@ -16,6 +20,8 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.annotation.Nullable;
/* /*
* Created by Christian Schabesberger on 02.02.16. * Created by Christian Schabesberger on 02.02.16.
* *
@ -36,17 +42,20 @@ import java.util.regex.Pattern;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})"); private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN
private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory(); = Pattern.compile("^([a-zA-Z0-9_-]{11})");
private static final List<String> SUBPATHS = Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/"); private static final YoutubeStreamLinkHandlerFactory INSTANCE
= new YoutubeStreamLinkHandlerFactory();
private static final List<String> SUBPATHS
= Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/");
private YoutubeStreamLinkHandlerFactory() { private YoutubeStreamLinkHandlerFactory() {
} }
public static YoutubeStreamLinkHandlerFactory getInstance() { public static YoutubeStreamLinkHandlerFactory getInstance() {
return instance; return INSTANCE;
} }
@Nullable @Nullable
@ -68,18 +77,21 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
@Override @Override
public String getUrl(String id) { public String getUrl(final String id) {
return "https://www.youtube.com/watch?v=" + id; return "https://www.youtube.com/watch?v=" + id;
} }
@Override @Override
public String getId(String urlString) throws ParsingException, IllegalArgumentException { public String getId(final String theUrlString)
throws ParsingException, IllegalArgumentException {
String urlString = theUrlString;
try { try {
URI uri = new URI(urlString); final URI uri = new URI(urlString);
String scheme = uri.getScheme(); final String scheme = uri.getScheme();
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { if (scheme != null
String schemeSpecificPart = uri.getSchemeSpecificPart(); && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
final String schemeSpecificPart = uri.getSchemeSpecificPart();
if (schemeSpecificPart.startsWith("//")) { if (schemeSpecificPart.startsWith("//")) {
final String extractedId = extractId(schemeSpecificPart.substring(2)); final String extractedId = extractId(schemeSpecificPart.substring(2));
if (extractedId != null) { if (extractedId != null) {
@ -91,26 +103,25 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(schemeSpecificPart); return assertIsId(schemeSpecificPart);
} }
} }
} catch (URISyntaxException ignored) { } catch (final URISyntaxException ignored) {
} }
URL url; final URL url;
try { try {
url = Utils.stringToURL(urlString); url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid"); throw new IllegalArgumentException("The given URL is not valid");
} }
String host = url.getHost(); final String host = url.getHost();
String path = url.getPath(); String path = url.getPath();
// remove leading "/" of URL-path if URL-path is given // remove leading "/" of URL-path if URL-path is given
if (!path.isEmpty()) { if (!path.isEmpty()) {
path = path.substring(1); path = path.substring(1);
} }
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) || if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url)
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) || || isHooktubeURL(url) || isInvidioURL(url) || isY2ubeURL(url))) {
YoutubeParsingHelper.isInvidioURL(url) || YoutubeParsingHelper.isY2ubeURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString); throw new FoundAdException("Error found ad: " + urlString);
} }
@ -122,16 +133,14 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
throw new ParsingException("Error no suitable url: " + urlString); throw new ParsingException("Error no suitable url: " + urlString);
} }
// using uppercase instead of lowercase, because toLowercase replaces some unicode characters // Using uppercase instead of lowercase, because toLowercase replaces some unicode
// with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls. // characters with their lowercase ASCII equivalent. Using toLowercase could result in
// faultily matching unicode urls.
switch (host.toUpperCase()) { switch (host.toUpperCase()) {
case "WWW.YOUTUBE-NOCOOKIE.COM": { case "WWW.YOUTUBE-NOCOOKIE.COM": {
if (path.startsWith("embed/")) { if (path.startsWith("embed/")) {
String id = path.substring(6); // embed/ return assertIsId(path.substring(6));
return assertIsId(id);
} }
break; break;
} }
@ -140,29 +149,31 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "M.YOUTUBE.COM": case "M.YOUTUBE.COM":
case "MUSIC.YOUTUBE.COM": { case "MUSIC.YOUTUBE.COM": {
if (path.equals("attribution_link")) { if (path.equals("attribution_link")) {
String uQueryValue = Utils.getQueryValue(url, "u"); final String uQueryValue = Utils.getQueryValue(url, "u");
URL decodedURL; final URL decodedURL;
try { try {
decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue); decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue);
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
throw new ParsingException("Error no suitable url: " + urlString); throw new ParsingException("Error no suitable url: " + urlString);
} }
String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); final String viewQueryValue = Utils.getQueryValue(decodedURL, "v");
return assertIsId(viewQueryValue); return assertIsId(viewQueryValue);
} }
String maybeId = getIdFromSubpathsInPath(path); final String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId; if (maybeId != null) {
return maybeId;
}
String viewQueryValue = Utils.getQueryValue(url, "v"); final String viewQueryValue = Utils.getQueryValue(url, "v");
return assertIsId(viewQueryValue); return assertIsId(viewQueryValue);
} }
case "Y2U.BE": case "Y2U.BE":
case "YOUTU.BE": { case "YOUTU.BE": {
String viewQueryValue = Utils.getQueryValue(url, "v"); final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) { if (viewQueryValue != null) {
return assertIsId(viewQueryValue); return assertIsId(viewQueryValue);
} }
@ -200,15 +211,17 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "YT.CYBERHOST.UK": case "YT.CYBERHOST.UK":
case "Y.COM.CM": { // code-block for hooktube.com and Invidious instances case "Y.COM.CM": { // code-block for hooktube.com and Invidious instances
if (path.equals("watch")) { if (path.equals("watch")) {
String viewQueryValue = Utils.getQueryValue(url, "v"); final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) { if (viewQueryValue != null) {
return assertIsId(viewQueryValue); return assertIsId(viewQueryValue);
} }
} }
String maybeId = getIdFromSubpathsInPath(path); final String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId; if (maybeId != null) {
return maybeId;
}
String viewQueryValue = Utils.getQueryValue(url, "v"); final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) { if (viewQueryValue != null) {
return assertIsId(viewQueryValue); return assertIsId(viewQueryValue);
} }
@ -225,17 +238,17 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
try { try {
getId(url); getId(url);
return true; return true;
} catch (FoundAdException fe) { } catch (final FoundAdException fe) {
throw fe; throw fe;
} catch (ParsingException e) { } catch (final ParsingException e) {
return false; return false;
} }
} }
private String getIdFromSubpathsInPath(String path) throws ParsingException { private String getIdFromSubpathsInPath(final String path) throws ParsingException {
for (final String subpath : SUBPATHS) { for (final String subpath : SUBPATHS) {
if (path.startsWith(subpath)) { if (path.startsWith(subpath)) {
String id = path.substring(subpath.length()); final String id = path.substring(subpath.length());
return assertIsId(id); return assertIsId(id);
} }
} }

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
View File

@ -20,8 +20,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException; import java.net.MalformedURLException;
@ -30,25 +32,28 @@ import java.util.List;
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
public String getUrl(String id, List<String> contentFilters, String sortFilter) { public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
return "https://www.youtube.com/feed/trending"; return "https://www.youtube.com/feed/trending";
} }
@Override @Override
public String getId(String url) { public String getId(final String url) {
return "Trending"; return "Trending";
} }
@Override @Override
public boolean onAcceptUrl(final String url) { public boolean onAcceptUrl(final String url) {
URL urlObj; final URL urlObj;
try { try {
urlObj = Utils.stringToURL(url); urlObj = Utils.stringToURL(url);
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
return false; return false;
} }
String urlPath = urlObj.getPath(); final String urlPath = urlObj.getPath();
return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending"); return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj))
&& urlPath.equals("/feed/trending");
} }
} }