be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Fix checkstyle issues

This commit is contained in:
Stypox 2022-03-18 15:09:06 +01:00 committed by litetex
parent 9dc17cd1ca
commit 740a37a2de
27 changed files with 684 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -1,14 +1,22 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA_OPUS;
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.schabi.newpipe.extractor.MediaFormat.*;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.*;
public class ItagItem {
/**
* List can be found here https://github.com/ytdl-org/youtube-dl/blob/9fc5eafb8e384453a49f7cfe73147be491f0b19d/youtube_dl/extractor/youtube.py#L1071
* List can be found here
* https://github.com/ytdl-org/youtube-dl/blob/9fc5eafb8e384453a49f7cfe73147be491f0b19d/youtube_dl/extractor/youtube.py#L1071
*/
private static final ItagItem[] ITAG_LIST = {
/////////////////////////////////////////////////////
@ -79,8 +87,8 @@ public class ItagItem {
// Utils
//////////////////////////////////////////////////////////////////////////*/
public static boolean isSupported(int itag) {
for (ItagItem item : ITAG_LIST) {
public static boolean isSupported(final int itag) {
for (final ItagItem item : ITAG_LIST) {
if (itag == item.id) {
return true;
}
@ -88,8 +96,8 @@ public class ItagItem {
return false;
}
public static ItagItem getItag(int itagId) throws ParsingException {
for (ItagItem item : ITAG_LIST) {
public static ItagItem getItag(final int itagId) throws ParsingException {
for (final ItagItem item : ITAG_LIST) {
if (itagId == item.id) {
return item;
}
@ -110,7 +118,10 @@ public class ItagItem {
/**
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final String resolution) {
this.id = id;
this.itagType = type;
this.mediaFormat = format;
@ -123,7 +134,11 @@ public class ItagItem {
*
* @param resolution string that will be used in the frontend
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final String resolution,
final int fps) {
this.id = id;
this.itagType = type;
this.mediaFormat = format;
@ -131,7 +146,10 @@ public class ItagItem {
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
public ItagItem(final int id,
final ItagType type,
final MediaFormat format,
final int avgBitrate) {
this.id = id;
this.itagType = type;
this.mediaFormat = format;
@ -170,7 +188,7 @@ public class ItagItem {
return bitrate;
}
public void setBitrate(int bitrate) {
public void setBitrate(final int bitrate) {
this.bitrate = bitrate;
}
@ -178,7 +196,7 @@ public class ItagItem {
return width;
}
public void setWidth(int width) {
public void setWidth(final int width) {
this.width = width;
}
@ -186,7 +204,7 @@ public class ItagItem {
return height;
}
public void setHeight(int height) {
public void setHeight(final int height) {
this.height = height;
}
@ -194,7 +212,7 @@ public class ItagItem {
return initStart;
}
public void setInitStart(int initStart) {
public void setInitStart(final int initStart) {
this.initStart = initStart;
}
@ -202,7 +220,7 @@ public class ItagItem {
return initEnd;
}
public void setInitEnd(int initEnd) {
public void setInitEnd(final int initEnd) {
this.initEnd = initEnd;
}
@ -210,7 +228,7 @@ public class ItagItem {
return indexStart;
}
public void setIndexStart(int indexStart) {
public void setIndexStart(final int indexStart) {
this.indexStart = indexStart;
}
@ -218,7 +236,7 @@ public class ItagItem {
return indexEnd;
}
public void setIndexEnd(int indexEnd) {
public void setIndexEnd(final int indexEnd) {
this.indexEnd = indexEnd;
}
@ -226,7 +244,7 @@ public class ItagItem {
return quality;
}
public void setQuality(String quality) {
public void setQuality(final String quality) {
this.quality = quality;
}
@ -234,7 +252,7 @@ public class ItagItem {
return codec;
}
public void setCodec(String codec) {
public void setCodec(final String codec) {
this.codec = codec;
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java
View File

@ -19,7 +19,7 @@ import javax.annotation.Nonnull;
* This class handling fetching the JavaScript file in order to allow other classes to extract the
* needed functions.
*/
public class YoutubeJavaScriptExtractor {
public final class YoutubeJavaScriptExtractor {
private static final String HTTPS = "https:";
private static String cachedJavaScriptCode;
@ -81,9 +81,10 @@ public class YoutubeJavaScriptExtractor {
final String hashPattern = "player\\\\\\/([a-z0-9]{8})\\\\\\/";
final String hash = Parser.matchGroup1(hashPattern, iframeContent);
return String.format("https://www.youtube.com/s/player/%s/player_ias.vflset/en_US/base.js", hash);
} catch (final Exception i) { }
return String.format(
"https://www.youtube.com/s/player/%s/player_ias.vflset/en_US/base.js", hash);
} catch (final Exception ignored) {
}
throw new ParsingException("Iframe API did not provide YouTube player js url");
}
@ -109,8 +110,8 @@ public class YoutubeJavaScriptExtractor {
}
}
}
} catch (final Exception i) { }
} catch (final Exception ignored) {
}
throw new ParsingException("Embedded info did not provide YouTube player js url");
}

200
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1,5 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
@ -10,7 +17,11 @@ import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
@ -28,18 +39,18 @@ import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 02.03.16.
*
@ -60,7 +71,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeParsingHelper {
public final class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
@ -99,10 +110,10 @@ public class YoutubeParsingHelper {
"https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static boolean isGoogleURL(String url) {
url = extractCachedUrlIfNeeded(url);
private static boolean isGoogleURL(final String url) {
final String cachedUrl = extractCachedUrlIfNeeded(url);
try {
final URL u = new URL(url);
final URL u = new URL(cachedUrl);
final String host = u.getHost();
return host.startsWith("google.")
|| host.startsWith("m.google.")
@ -442,7 +453,10 @@ public class YoutubeParsingHelper {
private static void extractClientVersionAndKey() throws IOException, ExtractionException {
// Don't extract the client version and the InnerTube key if it has been already extracted
if (keyAndVersionExtracted) return;
if (keyAndVersionExtracted) {
return;
}
// Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
final Map<String, List<String>> headers = new HashMap<>();
@ -460,8 +474,8 @@ public class YoutubeParsingHelper {
final JsonArray params = s.getArray("params");
for (final Object param : params) {
final JsonObject p = (JsonObject) param;
final String key = p.getString("key");
if (key != null && key.equals("cver")) {
final String paramKey = p.getString("key");
if (paramKey != null && paramKey.equals("cver")) {
clientVersion = p.getString("value");
}
}
@ -471,8 +485,8 @@ public class YoutubeParsingHelper {
final JsonArray params = s.getArray("params");
for (final Object param : params) {
final JsonObject p = (JsonObject) param;
final String key = p.getString("key");
if (key != null && key.equals("client.version")) {
final String paramKey = p.getString("key");
if (paramKey != null && paramKey.equals("client.version")) {
shortClientVersion = p.getString("value");
}
}
@ -516,9 +530,12 @@ public class YoutubeParsingHelper {
* Get the client version
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (!isNullOrEmpty(clientVersion)) {
return clientVersion;
}
if (areHardcodedClientVersionAndKeyValid()) {
return clientVersion = HARDCODED_CLIENT_VERSION;
clientVersion = HARDCODED_CLIENT_VERSION;
return clientVersion;
}
extractClientVersionAndKey();
@ -529,9 +546,12 @@ public class YoutubeParsingHelper {
* Get the key
*/
public static String getKey() throws IOException, ExtractionException {
if (!isNullOrEmpty(key)) return key;
if (!isNullOrEmpty(key)) {
return key;
}
if (areHardcodedClientVersionAndKeyValid()) {
return key = HARDCODED_KEY;
key = HARDCODED_KEY;
return key;
}
extractClientVersionAndKey();
@ -574,7 +594,7 @@ public class YoutubeParsingHelper {
+ HARDCODED_YOUTUBE_MUSIC_KEY[0];
// @formatter:off
byte[] json = JsonWriter.string()
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
@ -617,9 +637,12 @@ public class YoutubeParsingHelper {
public static String[] getYoutubeMusicKey() throws IOException, ReCaptchaException,
Parser.RegexException {
if (youtubeMusicKey != null && youtubeMusicKey.length == 3) return youtubeMusicKey;
if (youtubeMusicKey != null && youtubeMusicKey.length == 3) {
return youtubeMusicKey;
}
if (isHardcodedYoutubeMusicKeyValid()) {
return youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY;
youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY;
return youtubeMusicKey;
}
final String url = "https://music.youtube.com/";
@ -627,31 +650,36 @@ public class YoutubeParsingHelper {
addCookieHeader(headers);
final String html = getDownloader().get(url, headers).responseBody();
String key;
String innertubeApiKey;
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
innertubeApiKey = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (final Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
innertubeApiKey = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
}
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),",
html);
final String innertubeClientName
= Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
String clientVersion;
String innertubeClientVersion;
try {
clientVersion = Parser.matchGroup1(
innertubeClientVersion = Parser.matchGroup1(
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException e) {
try {
clientVersion = Parser.matchGroup1(
innertubeClientVersion = Parser.matchGroup1(
"INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1(
innertubeClientVersion = Parser.matchGroup1(
"innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
}
}
return youtubeMusicKey = new String[]{key, clientName, clientVersion};
youtubeMusicKey = new String[]{
innertubeApiKey,
innertubeClientName,
innertubeClientVersion
};
return youtubeMusicKey;
}
@Nullable
@ -667,16 +695,14 @@ public class YoutubeParsingHelper {
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
final String[] params = internUrl.split("&");
for (final String param : params) {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], UTF_8);
return URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (final UnsupportedEncodingException e) {
return null;
}
return url;
}
}
} else if (internUrl.startsWith("http")) {
@ -702,7 +728,7 @@ public class YoutubeParsingHelper {
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\""
+ browseEndpoint + "\")");
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
final StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
@ -715,8 +741,8 @@ public class YoutubeParsingHelper {
}
return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
return "https://www.youtube.com/playlist?list="
+ navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
}
return null;
}
@ -731,17 +757,23 @@ public class YoutubeParsingHelper {
@Nullable
public static String getTextFromObject(final JsonObject textObject, final boolean html)
throws ParsingException {
if (isNullOrEmpty(textObject)) return null;
if (isNullOrEmpty(textObject)) {
return null;
}
if (textObject.has("simpleText")) return textObject.getString("simpleText");
if (textObject.has("simpleText")) {
return textObject.getString("simpleText");
}
if (textObject.getArray("runs").isEmpty()) return null;
if (textObject.getArray("runs").isEmpty()) {
return null;
}
final StringBuilder textBuilder = new StringBuilder();
for (final Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
final String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text)
@ -768,27 +800,28 @@ public class YoutubeParsingHelper {
}
@Nullable
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String key)
public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey)
throws ParsingException {
if (jsonObject.isString(key)) {
return jsonObject.getString(key);
if (jsonObject.isString(theKey)) {
return jsonObject.getString(theKey);
} else {
return getTextFromObject(jsonObject.getObject(key));
return getTextFromObject(jsonObject.getObject(theKey));
}
}
public static String fixThumbnailUrl(@Nonnull String thumbnailUrl) {
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = thumbnailUrl.substring(2);
public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) {
String result = thumbnailUrl;
if (result.startsWith("//")) {
result = result.substring(2);
}
if (thumbnailUrl.startsWith(HTTP)) {
thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl);
} else if (!thumbnailUrl.startsWith(HTTPS)) {
thumbnailUrl = "https://" + thumbnailUrl;
if (result.startsWith(HTTP)) {
result = Utils.replaceHttpWithHttps(result);
} else if (!result.startsWith(HTTPS)) {
result = "https://" + result;
}
return thumbnailUrl;
return result;
}
public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem)
@ -882,7 +915,7 @@ public class YoutubeParsingHelper {
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization);
@ -1011,7 +1044,8 @@ public class YoutubeParsingHelper {
throws IOException, ExtractionException {
if (withThirdParty) {
// @formatter:off
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId)
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
@ -1070,7 +1104,7 @@ public class YoutubeParsingHelper {
*/
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
headers.put("Cookie", Collections.singletonList(generateConsentCookie()));
} else {
headers.get("Cookie").add(generateConsentCookie());
}
@ -1121,15 +1155,27 @@ public class YoutubeParsingHelper {
if (alertText.contains("violation") || alertText.contains("violating")
|| alertText.contains("infringement")) {
// Possible error messages:
// "This account has been terminated for a violation of YouTube's Terms of Service."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten."
// "This account has been terminated due to multiple or severe violations of YouTube's policy against spam, deceptive practices and misleading content or other Terms of Service violations."
// "This account has been terminated due to multiple or severe violations of YouTube's policy on nudity or sexual content."
// "This account has been terminated for violating YouTube's Community Guidelines."
// "This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted."
// "This account has been terminated because it is linked to an account that received multiple third-party claims of copyright infringement."
throw new AccountTerminatedException(alertText, AccountTerminatedException.Reason.VIOLATION);
// "This account has been terminated for a violation of YouTube's Terms of
// Service."
// "This account has been terminated due to multiple or severe violations of
// YouTube's policy prohibiting hate speech."
// "This account has been terminated due to multiple or severe violations of
// YouTube's policy prohibiting content designed to harass, bully or
// threaten."
// "This account has been terminated due to multiple or severe violations
// of YouTube's policy against spam, deceptive practices and misleading
// content or other Terms of Service violations."
// "This account has been terminated due to multiple or severe violations of
// YouTube's policy on nudity or sexual content."
// "This account has been terminated for violating YouTube's Community
// Guidelines."
// "This account has been terminated because we received multiple
// third-party claims of copyright infringement regarding material that
// the user posted."
// "This account has been terminated because it is linked to an account that
// received multiple third-party claims of copyright infringement."
throw new AccountTerminatedException(alertText,
AccountTerminatedException.Reason.VIOLATION);
} else {
throw new AccountTerminatedException(alertText);
}
@ -1146,8 +1192,8 @@ public class YoutubeParsingHelper {
for (final Object content : contents) {
final JsonObject resultObject = (JsonObject) content;
if (resultObject.has("itemSectionRenderer")) {
for (final Object sectionContentObject :
resultObject.getObject("itemSectionRenderer").getArray("contents")) {
for (final Object sectionContentObject
: resultObject.getObject("itemSectionRenderer").getArray("contents")) {
final JsonObject sectionContent = (JsonObject) sectionContentObject;
if (sectionContent.has("infoPanelContentRenderer")) {
@ -1200,8 +1246,8 @@ public class YoutubeParsingHelper {
}
@Nonnull
private static MetaInfo getClarificationRendererContent(@Nonnull final JsonObject clarificationRenderer)
throws ParsingException {
private static MetaInfo getClarificationRendererContent(
@Nonnull final JsonObject clarificationRenderer) throws ParsingException {
final MetaInfo metaInfo = new MetaInfo();
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
@ -1275,7 +1321,7 @@ public class YoutubeParsingHelper {
return false;
}
for (Object badge : badges) {
for (final Object badge : badges) {
final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer")
.getString("style");
if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED")

62
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,11 +1,16 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
@ -42,12 +47,6 @@ import java.util.List;
import javax.annotation.Nonnull;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
/*
* Created by Christian Schabesberger on 23.08.15.
*
@ -70,7 +69,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
public class YoutubeService extends StreamingService {
public YoutubeService(int id) {
public YoutubeService(final int id) {
super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS));
}
@ -100,12 +99,12 @@ public class YoutubeService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new YoutubeStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new YoutubeChannelExtractor(this, linkHandler);
}
@ -120,7 +119,7 @@ public class YoutubeService extends StreamingService {
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
final List<String> contentFilters = query.getContentFilters();
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) {
@ -137,22 +136,21 @@ public class YoutubeService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(this);
final KioskList list = new KioskList(this);
// add kiosks here e.g.:
try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
}
}, new YoutubeTrendingLinkHandlerFactory(), "Trending");
list.addKioskEntry(
(streamingService, url, id) -> new YoutubeTrendingExtractor(
YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
id
),
new YoutubeTrendingLinkHandlerFactory(),
"Trending"
);
list.setDefaultKiosk("Trending");
} catch (Exception e) {
} catch (final Exception e) {
throw new ExtractionException(e);
}
@ -176,7 +174,7 @@ public class YoutubeService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler)
public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler)
throws ExtractionException {
return new YoutubeCommentsExtractor(this, urlIdHandler);
}
@ -199,14 +197,14 @@ public class YoutubeService extends StreamingService {
// https://www.youtube.com/picker_ajax?action_country_json=1
private static final List<ContentCountry> SUPPORTED_COUNTRIES = ContentCountry.listFrom(
"DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA", "CL",
"CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR", "GE", "DE",
"GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL", "IT", "JM", "JP",
"JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY", "MT", "MX", "ME", "MA",
"NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA", "PG", "PY", "PE", "PH", "PL",
"PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG", "SK", "SI", "ZA", "KR", "ES", "LK",
"SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG", "UA", "AE", "GB", "US", "UY", "VE", "VN",
"YE", "ZW"
"DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA",
"CL", "CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR",
"GE", "DE", "GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL",
"IT", "JM", "JP", "JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY",
"MT", "MX", "ME", "MA", "NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA",
"PG", "PY", "PE", "PH", "PL", "PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG",
"SK", "SI", "ZA", "KR", "ES", "LK", "SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG",
"UA", "AE", "GB", "US", "UY", "VE", "VN", "YE", "ZW"
);
@Override

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View File

@ -80,7 +80,8 @@ public class YoutubeThrottlingDecrypter {
public static String apply(final String url, final String videoId) throws ParsingException {
if (containsNParam(url)) {
if (FUNCTION == null) {
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
final String playerJsCode
= YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
FUNCTION_NAME = parseDecodeFunctionName(playerJsCode);
FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
@ -118,19 +119,22 @@ public class YoutubeThrottlingDecrypter {
throws Parser.RegexException {
try {
return parseWithParenthesisMatching(playerJsCode, functionName);
} catch (Exception e) {
} catch (final Exception e) {
return parseWithRegex(playerJsCode, functionName);
}
}
@Nonnull
private static String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
private static String parseWithParenthesisMatching(final String playerJsCode,
final String functionName) {
final String functionBase = functionName + "=function";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
+ ";";
}
@Nonnull
private static String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
private static String parseWithRegex(final String playerJsCode, final String functionName)
throws Parser.RegexException {
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
Pattern.DOTALL);
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
@ -155,7 +159,9 @@ public class YoutubeThrottlingDecrypter {
return Parser.matchGroup1(N_PARAM_PATTERN, url);
}
private static String decryptNParam(final String function, final String functionName, final String nParam) {
private static String decryptNParam(final String function,
final String functionName,
final String nParam) {
if (N_PARAMS_CACHE.containsKey(nParam)) {
return N_PARAMS_CACHE.get(nParam);
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,5 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
@ -31,11 +43,6 @@ import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 25.07.16.
*
@ -56,13 +63,12 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private JsonObject videoTab;
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it.
* Some channels have response redirects and the only way to reliably get the id is by saving it
* <p>
* "Movies & Shows":
* <pre>
@ -233,7 +239,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
String url = initialData.getObject("header")
final String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url");
@ -246,7 +252,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
String url = initialData.getObject("header")
final String url = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails")
.getObject(0).getString("url");
@ -361,7 +367,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
@ -436,28 +442,30 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab;
if (this.videoTab != null) {
return this.videoTab;
}
JsonArray tabs = initialData.getObject("contents")
final JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject videoTab = null;
JsonObject foundVideoTab = null;
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");
foundVideoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
}
}
if (videoTab == null) {
if (foundVideoTab == null) {
throw new ContentNotSupportedException("This channel has no Videos tab");
}
final String messageRendererText = getTextFromObject(videoTab.getObject("content")
final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
@ -466,7 +474,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return null;
}
this.videoTab = videoTab;
return videoTab;
this.videoTab = foundVideoTab;
return foundVideoTab;
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -33,19 +33,20 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
*/
public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JsonObject channelInfoItem;
private final JsonObject channelInfoItem;
public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) {
public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) {
this.channelInfoItem = channelInfoItem;
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
final String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails")
.getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@ -54,7 +55,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
public String getName() throws ParsingException {
try {
return getTextFromObject(channelInfoItem.getObject("title"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get name", e);
}
}
@ -62,9 +63,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getUrl() throws ParsingException {
try {
String id = "channel/" + channelInfoItem.getString("channelId");
final String id = "channel/" + channelInfoItem.getString("channelId");
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get url", e);
}
}
@ -77,8 +78,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
return -1;
}
return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText")));
} catch (Exception e) {
return Utils.mixedNumberWordToLong(getTextFromObject(
channelInfoItem.getObject("subscriberCountText")));
} catch (final Exception e) {
throw new ParsingException("Could not get subscriber count", e);
}
}
@ -93,7 +95,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
channelInfoItem.getObject("videoCountText"))));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}
}
@ -112,7 +114,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
}
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get description", e);
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -103,7 +103,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
itemSectionRenderer
.getObject("itemSectionRenderer")
.getArray("contents").getObject(0),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
"continuationItemRenderer.continuationEndpoint"
+ ".continuationCommand.token");
} catch (final ParsingException ignored) {
return null;
}

40
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -1,9 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -14,9 +16,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json;
@ -33,11 +32,12 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
private JsonObject getCommentRenderer() throws ParsingException {
if(commentRenderer == null) {
if(!json.has("comment"))
commentRenderer = json;
else
if (commentRenderer == null) {
if (json.has("comment")) {
commentRenderer = JsonUtils.getObject(json, "comment.commentRenderer");
} else {
commentRenderer = json;
}
}
return commentRenderer;
}
@ -50,7 +50,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails");
final JsonArray arr = JsonUtils.getArray(getCommentRenderer(),
"authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url");
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
@ -69,7 +70,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getTextualUploadDate() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "publishedTimeText"));
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(),
"publishedTimeText"));
} catch (final Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
@ -78,7 +80,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
String textualPublishedTime = getTextualUploadDate();
final String textualPublishedTime = getTextualUploadDate();
if (timeAgoParser != null && textualPublishedTime != null
&& !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime);
@ -108,7 +110,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
final String likeCount;
try {
likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(getCommentRenderer(),
"actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer.accessibilityData.accessibilityData.label"));
"actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer"
+ ".accessibilityData.accessibilityData.label"));
} catch (final Exception e) {
// Use the approximate like count returned into the voteCount object
// This may return a language dependent version, e.g. in German: 3,3 Mio
@ -202,7 +205,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails");
final JsonArray arr = JsonUtils.getArray(getCommentRenderer(),
"authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url");
} catch (final Exception e) {
throw new ParsingException("Could not get author thumbnail", e);
@ -211,7 +215,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public boolean isHeartedByUploader() throws ParsingException {
final JsonObject commentActionButtonsRenderer = getCommentRenderer().getObject("actionButtons")
final JsonObject commentActionButtonsRenderer = getCommentRenderer()
.getObject("actionButtons")
.getObject("commentActionButtonsRenderer");
return commentActionButtonsRenderer.has("creatorHeart");
}
@ -247,10 +252,13 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public Page getReplies() throws ParsingException {
try {
final String id = JsonUtils.getString(JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents").getObject(0), "continuationItemRenderer.continuationEndpoint.continuationCommand.token");
final String id = JsonUtils.getString(
JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents")
.getObject(0),
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
return new Page(url, id);
} catch (final Exception e) {
return null; // Would return null for Comment Replies, since YouTube does not support nested replies.
return null;
}
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java
View File

@ -22,14 +22,15 @@ import java.io.IOException;
import javax.annotation.Nonnull;
public class YoutubeFeedExtractor extends FeedExtractor {
public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) {
public YoutubeFeedExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
private Document document;
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelIdOrUser = getLinkHandler().getId();
final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
@ -46,7 +47,7 @@ public class YoutubeFeedExtractor extends FeedExtractor {
final Elements entries = document.select("feed > entry");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
for (Element entryElement : entries) {
for (final Element entryElement : entries) {
collector.commit(new YoutubeFeedInfoItemExtractor(entryElement));
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java
View File

@ -13,7 +13,7 @@ import java.time.format.DateTimeParseException;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement;
public YoutubeFeedInfoItemExtractor(Element entryElement) {
public YoutubeFeedInfoItemExtractor(final Element entryElement) {
this.entryElement = entryElement;
}
@ -37,7 +37,8 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public long getViewCount() {
return Long.parseLong(entryElement.getElementsByTag("media:statistics").first().attr("views"));
return Long.parseLong(entryElement.getElementsByTag("media:statistics").first()
.attr("views"));
}
@Override
@ -72,8 +73,9 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
public DateWrapper getUploadDate() throws ParsingException {
try {
return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate()));
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
} catch (final DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")",
e);
}
}

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -1,10 +1,21 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -25,14 +36,14 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
/**
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
* It handles URLs in the format of
@ -84,8 +95,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
if (isNullOrEmpty(playlistData)) throw new ExtractionException(
"Could not get playlistData");
if (isNullOrEmpty(playlistData)) {
throw new ExtractionException("Could not get playlistData");
}
cookieValue = extractCookieValue(COOKIE_NAME, response);
}

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -1,6 +1,24 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -19,17 +37,14 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
@ -162,7 +177,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return false;
}
JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
final JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
return firstContent.has("didYouMeanRenderer")
|| firstContent.has("showingResultsForRenderer");
@ -211,7 +226,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
// @formatter:off
byte[] json = JsonWriter.string()
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
@ -331,7 +346,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
if (searchType.equals(MUSIC_VIDEOS)) {
JsonArray items = info.getObject("menu").getObject("menuRenderer")
final JsonArray items = info.getObject("menu")
.getObject("menuRenderer")
.getArray("items");
for (final Object item : items) {
final JsonObject menuNavigationItemRenderer =
@ -354,8 +370,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0);
if (!navigationEndpointHolder.has("navigationEndpoint"))
if (!navigationEndpointHolder.has("navigationEndpoint")) {
return null;
}
final String url = getUrlFromNavigationEndpoint(
navigationEndpointHolder.getObject("navigationEndpoint"));

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -1,5 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
@ -30,9 +42,6 @@ import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubePlaylistExtractor extends PlaylistExtractor {
// Minimum size of the stats array in the browse response which includes the streams count
private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2;

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -11,20 +11,20 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private JsonObject playlistInfoItem;
private final JsonObject playlistInfoItem;
public YoutubePlaylistInfoItemExtractor(JsonObject playlistInfoItem) {
public YoutubePlaylistInfoItemExtractor(final JsonObject playlistInfoItem) {
this.playlistInfoItem = playlistInfoItem;
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url = playlistInfoItem.getArray("thumbnails").getObject(0)
final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@ -33,7 +33,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
public String getName() throws ParsingException {
try {
return getTextFromObject(playlistInfoItem.getObject("title"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get name", e);
}
}
@ -41,9 +41,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getUrl() throws ParsingException {
try {
String id = playlistInfoItem.getString("playlistId");
final String id = playlistInfoItem.getString("playlistId");
return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get url", e);
}
}
@ -52,7 +52,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get uploader name", e);
}
}
@ -60,8 +60,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public long getStreamCount() throws ParsingException {
try {
return Long.parseLong(Utils.removeNonDigitCharacters(playlistInfoItem.getString("videoCount")));
} catch (Exception e) {
return Long.parseLong(Utils.removeNonDigitCharacters(
playlistInfoItem.getString("videoCount")));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}
}

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -1,6 +1,22 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -16,15 +32,11 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
/*
* Created by Christian Schabesberger on 22.07.2018
@ -179,7 +191,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}

149
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1,5 +1,18 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createPlayerBodyWithSts;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonMobilePostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileEmbedVideoJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopEmbedVideoJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
@ -7,7 +20,6 @@ import com.grack.nanojson.JsonWriter;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
@ -32,24 +44,35 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/*
* Created by Christian Schabesberger on 06.08.15.
@ -127,7 +150,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (isNullOrEmpty(title)) {
title = playerResponse.getObject("videoDetails").getString("title");
if (isNullOrEmpty(title)) throw new ParsingException("Could not get name");
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get name");
}
}
return title;
@ -158,8 +183,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.substring(10);
final String time = getTextFromObject(
getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
@ -206,10 +231,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getThumbnailUrl() throws ParsingException {
assertPageFetched();
try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail")
.getArray("thumbnails");
final JsonArray thumbnails = playerResponse.getObject("videoDetails")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (final Exception e) {
@ -224,9 +249,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
// Description with more info on links
try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer()
final String description = getTextFromObject(getVideoSecondaryInfoRenderer()
.getObject("description"), true);
if (!isNullOrEmpty(description)) return new Description(description, Description.HTML);
if (!isNullOrEmpty(description)) {
return new Description(description, Description.HTML);
}
} catch (final ParsingException ignored) {
// Age-restricted videos cause a ParsingException here
}
@ -327,10 +354,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (isNullOrEmpty(views)) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count");
if (isNullOrEmpty(views)) {
throw new ParsingException("Could not get view count");
}
}
if (views.toLowerCase().contains("no views")) return 0;
if (views.toLowerCase().contains("no views")) {
return 0;
}
return Long.parseLong(Utils.removeNonDigitCharacters(views));
}
@ -354,7 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (likesString == null) {
// If this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the like button is missing");
throw new ParsingException(
"Ratings are enabled even though the like button is missing");
}
return -1;
}
@ -400,7 +432,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// The difference between the real name of the channel and the displayed name is especially
// visible for music channels and autogenerated channels.
final String uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name");
if (isNullOrEmpty(uploaderName)) {
throw new ParsingException("Could not get uploader name");
}
return uploaderName;
}
@ -440,12 +474,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getUploaderSubscriberCount() throws ParsingException {
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer");
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer,
"owner.videoOwnerRenderer");
if (!videoOwnerRenderer.has("subscriberCountText")) {
return UNKNOWN_SUBSCRIBER_COUNT;
}
try {
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")));
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer
.getObject("subscriberCountText")));
} catch (final NumberFormatException e) {
throw new ParsingException("Could not get uploader subscriber count", e);
}
@ -677,7 +713,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
private static final String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
+ "\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
"\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)",
"\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)",
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;",
@ -720,7 +757,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
.contains("age");
if (!playerResponse.has("streamingData")) {
@ -765,19 +802,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
@Nonnull JsonObject playabilityStatus)
@Nonnull final JsonObject playabilityStatus)
throws ParsingException {
String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw the specific exception based on error message
// or a ContentNotAvailableException with the reason text if it's an unknown reason.
if (status != null && !status.equalsIgnoreCase("ok")) {
playabilityStatus = youtubePlayerResponse.getObject("playabilityStatus");
status = playabilityStatus.getString("status");
final String reason = playabilityStatus.getString("reason");
final JsonObject newPlayabilityStatus
= youtubePlayerResponse.getObject("playabilityStatus");
status = newPlayabilityStatus.getString("status");
final String reason = newPlayabilityStatus.getString("reason");
if (status.equalsIgnoreCase("login_required")) {
if (reason == null) {
final String message = playabilityStatus.getArray("messages").getString(0);
final String message = newPlayabilityStatus.getArray("messages").getString(0);
if (message != null && message.contains("private")) {
throw new PrivateContentException("This video is private.");
}
@ -797,11 +835,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PaidContentException("This video is a paid video");
}
if (reason.contains("members-only")) {
throw new PaidContentException(
"This video is only available for members of the channel of this video");
throw new PaidContentException("This video is only available"
+ " for members of the channel of this video");
}
if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(playabilityStatus
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("subreason"));
if (detailedErrorMessage != null) {
@ -900,8 +938,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String videoId)
throws IOException, ExtractionException {
final byte[] androidMobileEmbedBody = JsonWriter.string(
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.done())
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.done())
.getBytes(UTF_8);
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
androidMobileEmbedBody, contentCountry, localization);
@ -953,11 +991,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return false;
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
private String getDeobfuscationFuncName(final String thePlayerCode)
throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
return Parser.matchGroup1(regex, playerCode);
return Parser.matchGroup1(regex, thePlayerCode);
} catch (final Parser.RegexException re) {
if (exception == null) {
exception = re;
@ -1011,7 +1050,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private void getStsFromPlayerJs() throws ParsingException {
if (!isNullOrEmpty(sts)) return;
if (!isNullOrEmpty(sts)) {
return;
}
if (playerCode == null) {
storePlayerJs();
if (playerCode == null) {
@ -1045,51 +1086,55 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//////////////////////////////////////////////////////////////////////////*/
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
if (this.videoPrimaryInfoRenderer != null) {
return this.videoPrimaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoPrimaryInfoRenderer = null;
JsonObject theVideoPrimaryInfoRenderer = null;
for (final Object content : contents) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content)
theVideoPrimaryInfoRenderer = ((JsonObject) content)
.getObject("videoPrimaryInfoRenderer");
break;
}
}
if (isNullOrEmpty(videoPrimaryInfoRenderer)) {
if (isNullOrEmpty(theVideoPrimaryInfoRenderer)) {
throw new ParsingException("Could not find videoPrimaryInfoRenderer");
}
this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer;
return videoPrimaryInfoRenderer;
this.videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
return theVideoPrimaryInfoRenderer;
}
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
if (this.videoSecondaryInfoRenderer != null) {
return this.videoSecondaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoSecondaryInfoRenderer = null;
JsonObject theVideoSecondaryInfoRenderer = null;
for (final Object content : contents) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content)
theVideoSecondaryInfoRenderer = ((JsonObject) content)
.getObject("videoSecondaryInfoRenderer");
break;
}
}
if (isNullOrEmpty(videoSecondaryInfoRenderer)) {
if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) {
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
}
this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer;
return videoSecondaryInfoRenderer;
this.videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
return theVideoSecondaryInfoRenderer;
}
@Nonnull
@ -1120,8 +1165,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (streamingData != null && streamingData.has(streamingDataKey)) {
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
final JsonObject formatData = formats.getObject(i);
final int itag = formatData.getInt("itag");
if (ItagItem.isSupported(itag)) {
try {

89
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -1,7 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -12,15 +19,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nullable;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -51,7 +55,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
* @param videoInfoItem The JSON page element
* @param timeAgoParser A parser of the textual dates or {@code null}.
*/
public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) {
public YoutubeStreamInfoItemExtractor(final JsonObject videoInfoItem,
@Nullable final TimeAgoParser timeAgoParser) {
this.videoInfo = videoInfoItem;
this.timeAgoParser = timeAgoParser;
}
@ -64,43 +69,51 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
final JsonArray badges = videoInfo.getArray("badges");
for (final Object badge : badges) {
final JsonObject badgeRenderer = ((JsonObject) badge).getObject("metadataBadgeRenderer");
if (badgeRenderer.getString("style", EMPTY_STRING).equals("BADGE_STYLE_TYPE_LIVE_NOW") ||
badgeRenderer.getString("label", EMPTY_STRING).equals("LIVE NOW")) {
return cachedStreamType = StreamType.LIVE_STREAM;
final JsonObject badgeRenderer
= ((JsonObject) badge).getObject("metadataBadgeRenderer");
if (badgeRenderer.getString("style", EMPTY_STRING).equals("BADGE_STYLE_TYPE_LIVE_NOW")
|| badgeRenderer.getString("label", EMPTY_STRING).equals("LIVE NOW")) {
cachedStreamType = StreamType.LIVE_STREAM;
return cachedStreamType;
}
}
for (final Object overlay : videoInfo.getArray("thumbnailOverlays")) {
final String style = ((JsonObject) overlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING);
.getObject("thumbnailOverlayTimeStatusRenderer")
.getString("style", EMPTY_STRING);
if (style.equalsIgnoreCase("LIVE")) {
return cachedStreamType = StreamType.LIVE_STREAM;
cachedStreamType = StreamType.LIVE_STREAM;
return cachedStreamType;
}
}
return cachedStreamType = StreamType.VIDEO_STREAM;
cachedStreamType = StreamType.VIDEO_STREAM;
return cachedStreamType;
}
@Override
public boolean isAd() throws ParsingException {
return isPremium() || getName().equals("[Private video]") || getName().equals("[Deleted video]");
return isPremium() || getName().equals("[Private video]")
|| getName().equals("[Deleted video]");
}
@Override
public String getUrl() throws ParsingException {
try {
String videoId = videoInfo.getString("videoId");
final String videoId = videoInfo.getString("videoId");
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get url", e);
}
}
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(videoInfo.getObject("title"));
if (!isNullOrEmpty(name)) return name;
final String name = getTextFromObject(videoInfo.getObject("title"));
if (!isNullOrEmpty(name)) {
return name;
}
throw new ParsingException("Could not get name");
}
@ -113,14 +126,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
String duration = getTextFromObject(videoInfo.getObject("lengthText"));
if (isNullOrEmpty(duration)) {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
for (final Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
}
if (isNullOrEmpty(duration)) throw new ParsingException("Could not get duration");
if (isNullOrEmpty(duration)) {
throw new ParsingException("Could not get duration");
}
}
return YoutubeParsingHelper.parseDurationString(duration);
@ -136,7 +151,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (isNullOrEmpty(name)) {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
if (isNullOrEmpty(name)) throw new ParsingException("Could not get uploader name");
if (isNullOrEmpty(name)) {
throw new ParsingException("Could not get uploader name");
}
}
}
@ -156,7 +173,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
if (isNullOrEmpty(url)) throw new ParsingException("Could not get uploader url");
if (isNullOrEmpty(url)) {
throw new ParsingException("Could not get uploader url");
}
}
}
@ -168,7 +187,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getUploaderAvatarUrl() throws ParsingException {
if (videoInfo.has("channelThumbnailSupportedRenderers")) {
return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails")
return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers"
+ ".channelThumbnailWithLinkRenderer.thumbnail.thumbnails")
.getObject(0).getString("url");
}
@ -196,8 +216,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText;
final String publishedTimeText
= getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (publishedTimeText != null && !publishedTimeText.isEmpty()) {
return publishedTimeText;
}
return null;
}
@ -217,7 +240,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (timeAgoParser != null && !isNullOrEmpty(textualUploadDate)) {
try {
return timeAgoParser.parse(textualUploadDate);
} catch (ParsingException e) {
} catch (final ParsingException e) {
throw new ParsingException("Could not get upload date", e);
}
}
@ -245,7 +268,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get view count", e);
}
}
@ -256,9 +279,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
private boolean isPremium() {
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) {
final JsonArray badges = videoInfo.getArray("badges");
for (final Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer")
.getString("label", EMPTY_STRING).equals("Premium")) {
return true;
}
}
@ -276,8 +300,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try {
return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
ZoneOffset.UTC);
} catch (Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
} catch (final Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
}
}
@ -286,7 +310,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getShortDescription() throws ParsingException {
if (videoInfo.has("detailedMetadataSnippets")) {
return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText"));
return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets")
.getObject(0).getObject("snippetText"));
}
if (videoInfo.has("descriptionSnippet")) {

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View File

@ -70,7 +70,7 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
final JsonArray subscriptions;
try {
subscriptions = JsonParser.array().from(contentInputStream);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new InvalidSourceException("Invalid json input stream", e);
}
@ -101,7 +101,7 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
public List<SubscriptionItem> fromZipInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException {
try (final ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) {
try (ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) {
ZipEntry zipEntry;
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (zipEntry.getName().toLowerCase().endsWith(".csv")) {
@ -122,15 +122,16 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
throw new InvalidSourceException("Error reading contents of zip file", e);
}
throw new InvalidSourceException("Unable to find a valid subscriptions.csv file (try extracting and selecting the csv file)");
throw new InvalidSourceException("Unable to find a valid subscriptions.csv file"
+ " (try extracting and selecting the csv file)");
}
public List<SubscriptionItem> fromCsvInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException {
// Expected format of CSV file:
// Channel Id,Channel Url,Channel Title
// UC1JTQBa5QxZCpXrFSkMxmPw,http://www.youtube.com/channel/UC1JTQBa5QxZCpXrFSkMxmPw,Raycevick
// UCFl7yKfcRcFmIUbKeCA-SJQ,http://www.youtube.com/channel/UCFl7yKfcRcFmIUbKeCA-SJQ,Joji
// Channel Id,Channel Url,Channel Title
//UC1JTQBa5QxZCpXrFSkMxmPw,http://www.youtube.com/channel/UC1JTQBa5QxZCpXrFSkMxmPw,Raycevick
//UCFl7yKfcRcFmIUbKeCA-SJQ,http://www.youtube.com/channel/UCFl7yKfcRcFmIUbKeCA-SJQ,Joji
//
// Notes:
// It's always 3 columns

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -1,8 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -12,10 +16,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/*
* Created by Christian Schabesberger on 28.09.16.
@ -39,12 +43,12 @@ import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public YoutubeSuggestionExtractor(StreamingService service) {
public YoutubeSuggestionExtractor(final StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
public List<String> suggestionList(final String query) throws IOException, ExtractionException {
final Downloader dl = NewPipe.getDownloader();
final List<String> suggestions = new ArrayList<>();
@ -62,16 +66,20 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
// trim JSONP part "JP(...)"
response = response.substring(3, response.length() - 1);
try {
JsonArray collection = JsonParser.array().from(response).getArray(1);
for (Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) continue;
String suggestionStr = ((JsonArray) suggestion).getString(0);
if (suggestionStr == null) continue;
final JsonArray collection = JsonParser.array().from(response).getArray(1);
for (final Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) {
continue;
}
final String suggestionStr = ((JsonArray) suggestion).getString(0);
if (suggestionStr == null) {
continue;
}
suggestions.add(suggestionStr);
}
return suggestions;
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -55,7 +55,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
// @formatter:off
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
@ -92,15 +93,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonArray itemSectionRenderers = initialData.getObject("contents")
final JsonArray itemSectionRenderers = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents");
for (final Object itemSectionRenderer : itemSectionRenderers) {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer)
final JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View File

@ -29,30 +29,34 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory();
private static final YoutubeChannelLinkHandlerFactory INSTANCE
= new YoutubeChannelLinkHandlerFactory();
private static final Pattern excludedSegments =
Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site");
private static final Pattern EXCLUDED_SEGMENTS =
Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site");
private YoutubeChannelLinkHandlerFactory() {
}
public static YoutubeChannelLinkHandlerFactory getInstance() {
return instance;
return INSTANCE;
}
/**
* Returns URL to channel from an ID
*
* @param id Channel ID including e.g. 'channel/'
* @param contentFilters
* @param searchFilter
* @return URL to channel
*/
@Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) {
public String getUrl(final String id,
final List<String> contentFilters,
final String searchFilter) {
return "https://www.youtube.com/" + id;
}
/**
* Returns true if path conform to
* custom short channel URLs like youtube.com/yourcustomname
@ -61,17 +65,18 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
* @return true - if value conform to short channel URL, false - not
*/
private boolean isCustomShortChannelUrl(final String[] splitPath) {
return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches();
return splitPath.length == 1 && !EXCLUDED_SEGMENTS.matcher(splitPath[0]).matches();
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
try {
final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) {
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj)
|| YoutubeParsingHelper.isHooktubeURL(urlObj))) {
throw new ParsingException("the URL given is not a Youtube-URL");
}
@ -85,7 +90,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
splitPath = path.split("/");
}
if (!path.startsWith("user/") && !path.startsWith("channel/") && !path.startsWith("c/")) {
if (!path.startsWith("user/")
&& !path.startsWith("channel/")
&& !path.startsWith("c/")) {
throw new ParsingException("the URL given is neither a channel nor an user");
}
@ -97,15 +104,16 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
return splitPath[0] + "/" + id;
} catch (final Exception exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception);
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
exception);
}
}
@Override
public boolean onAcceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
return true;

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View File

@ -6,22 +6,27 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeCommentsLinkHandlerFactory instance = new YoutubeCommentsLinkHandlerFactory();
private static final YoutubeCommentsLinkHandlerFactory INSTANCE
= new YoutubeCommentsLinkHandlerFactory();
private YoutubeCommentsLinkHandlerFactory() {
}
public static YoutubeCommentsLinkHandlerFactory getInstance() {
return instance;
return INSTANCE;
}
@Override
public String getUrl(String id) {
public String getUrl(final String id) {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
public String getId(String urlString) throws ParsingException, IllegalArgumentException {
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); //we need the same id, avoids duplicate code
public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
// we need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
}
@Override
@ -29,15 +34,17 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
try {
getId(url);
return true;
} catch (FoundAdException fe) {
} catch (final FoundAdException fe) {
throw fe;
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return getUrl(id);
}
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -11,11 +11,14 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
new YoutubePlaylistLinkHandlerFactory();
private YoutubePlaylistLinkHandlerFactory() {
}
public static YoutubePlaylistLinkHandlerFactory getInstance() {
return INSTANCE;
}
@ -54,8 +57,10 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
&& Utils.getQueryValue(urlObj, "v") == null) {
//Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId
throw new ContentNotSupportedException("Channel Mix without a video id are not supported");
// Video id can't be determined from the channel mix id.
// See YoutubeParsingHelper#extractVideoIdFromMixId
throw new ContentNotSupportedException(
"Channel Mix without a video id are not supported");
}
return listID;
@ -69,15 +74,15 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
return true;
}
/**
* If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is like
* {@code https://youtube.com/watch?v=videoId&list=playlistId}
* If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is
* like {@code https://youtube.com/watch?v=videoId&list=playlistId}
* <p>Otherwise use super</p>
*/
@Override
@ -96,7 +101,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
getContentFilter(url),
getSortFilter(url));
}
} catch (MalformedURLException exception) {
} catch (final MalformedURLException exception) {
throw new ParsingException("Error could not parse URL: " + exception.getMessage(),
exception);
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -11,7 +11,7 @@ import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String VIDEOS = "videos";
@ -84,7 +84,10 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Nonnull
public static String getSearchParameter(final String contentFilter) {
if (isNullOrEmpty(contentFilter)) return "";
if (isNullOrEmpty(contentFilter)) {
return "";
}
switch (contentFilter) {
case VIDEOS:
return "EgIQAQ%3D%3D";

97
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View File

@ -1,12 +1,16 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
@ -16,6 +20,8 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
/*
* Created by Christian Schabesberger on 02.02.16.
*
@ -36,17 +42,20 @@ import java.util.regex.Pattern;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})");
private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
private static final List<String> SUBPATHS = Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/");
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN
= Pattern.compile("^([a-zA-Z0-9_-]{11})");
private static final YoutubeStreamLinkHandlerFactory INSTANCE
= new YoutubeStreamLinkHandlerFactory();
private static final List<String> SUBPATHS
= Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/");
private YoutubeStreamLinkHandlerFactory() {
}
public static YoutubeStreamLinkHandlerFactory getInstance() {
return instance;
return INSTANCE;
}
@Nullable
@ -68,18 +77,21 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(String id) {
public String getUrl(final String id) {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
public String getId(String urlString) throws ParsingException, IllegalArgumentException {
public String getId(final String theUrlString)
throws ParsingException, IllegalArgumentException {
String urlString = theUrlString;
try {
URI uri = new URI(urlString);
String scheme = uri.getScheme();
final URI uri = new URI(urlString);
final String scheme = uri.getScheme();
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
String schemeSpecificPart = uri.getSchemeSpecificPart();
if (scheme != null
&& (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
final String schemeSpecificPart = uri.getSchemeSpecificPart();
if (schemeSpecificPart.startsWith("//")) {
final String extractedId = extractId(schemeSpecificPart.substring(2));
if (extractedId != null) {
@ -91,26 +103,25 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(schemeSpecificPart);
}
}
} catch (URISyntaxException ignored) {
} catch (final URISyntaxException ignored) {
}
URL url;
final URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String host = url.getHost();
final String host = url.getHost();
String path = url.getPath();
// remove leading "/" of URL-path if URL-path is given
if (!path.isEmpty()) {
path = path.substring(1);
}
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
YoutubeParsingHelper.isInvidioURL(url) || YoutubeParsingHelper.isY2ubeURL(url))) {
if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url)
|| isHooktubeURL(url) || isInvidioURL(url) || isY2ubeURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString);
}
@ -122,16 +133,14 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
throw new ParsingException("Error no suitable url: " + urlString);
}
// using uppercase instead of lowercase, because toLowercase replaces some unicode characters
// with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls.
// Using uppercase instead of lowercase, because toLowercase replaces some unicode
// characters with their lowercase ASCII equivalent. Using toLowercase could result in
// faultily matching unicode urls.
switch (host.toUpperCase()) {
case "WWW.YOUTUBE-NOCOOKIE.COM": {
if (path.startsWith("embed/")) {
String id = path.substring(6); // embed/
return assertIsId(id);
return assertIsId(path.substring(6));
}
break;
}
@ -140,29 +149,31 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "M.YOUTUBE.COM":
case "MUSIC.YOUTUBE.COM": {
if (path.equals("attribution_link")) {
String uQueryValue = Utils.getQueryValue(url, "u");
final String uQueryValue = Utils.getQueryValue(url, "u");
URL decodedURL;
final URL decodedURL;
try {
decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new ParsingException("Error no suitable url: " + urlString);
}
String viewQueryValue = Utils.getQueryValue(decodedURL, "v");
final String viewQueryValue = Utils.getQueryValue(decodedURL, "v");
return assertIsId(viewQueryValue);
}
String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId;
final String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) {
return maybeId;
}
String viewQueryValue = Utils.getQueryValue(url, "v");
final String viewQueryValue = Utils.getQueryValue(url, "v");
return assertIsId(viewQueryValue);
}
case "Y2U.BE":
case "YOUTU.BE": {
String viewQueryValue = Utils.getQueryValue(url, "v");
final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsId(viewQueryValue);
}
@ -200,15 +211,17 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "YT.CYBERHOST.UK":
case "Y.COM.CM": { // code-block for hooktube.com and Invidious instances
if (path.equals("watch")) {
String viewQueryValue = Utils.getQueryValue(url, "v");
final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsId(viewQueryValue);
}
}
String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId;
final String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) {
return maybeId;
}
String viewQueryValue = Utils.getQueryValue(url, "v");
final String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsId(viewQueryValue);
}
@ -225,17 +238,17 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
try {
getId(url);
return true;
} catch (FoundAdException fe) {
} catch (final FoundAdException fe) {
throw fe;
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
}
private String getIdFromSubpathsInPath(String path) throws ParsingException {
private String getIdFromSubpathsInPath(final String path) throws ParsingException {
for (final String subpath : SUBPATHS) {
if (path.startsWith(subpath)) {
String id = path.substring(subpath.length());
final String id = path.substring(subpath.length());
return assertIsId(id);
}
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
View File

@ -20,8 +20,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
@ -30,25 +32,28 @@ import java.util.List;
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
public String getUrl(String id, List<String> contentFilters, String sortFilter) {
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
return "https://www.youtube.com/feed/trending";
}
@Override
public String getId(String url) {
public String getId(final String url) {
return "Trending";
}
@Override
public boolean onAcceptUrl(final String url) {
URL urlObj;
final URL urlObj;
try {
urlObj = Utils.stringToURL(url);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
return false;
}
String urlPath = urlObj.getPath();
return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending");
final String urlPath = urlObj.getPath();
return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj))
&& urlPath.equals("/feed/trending");
}
}