[YouTube] Add support for channel tabs and tags and age-restricted channels

Support of tags and videos, shorts, live, playlists and channels tabs has been
added for non-age restricted channels.

Age-restricted channels are now also supported and always returned the videos,
shorts and live tabs, accessible using system playlists. These tabs are the
only ones which can be accessed using YouTube's desktop website without being
logged-in.

The videos channel tab parameter has been updated to the one used by the
desktop website and when a channel extraction is fetched, this tab is returned
in the list of tabs as a cached one in the corresponding link handler.

Visitor data support per request has been added, as a valid visitor data is
required to fetch continuations with contents on the shorts tab. It is only
used in this case to enhance privacy.

A dedicated shorts UI elements (reelItemRenderers) extractor has been added,
YoutubeReelInfoItemExtractor. These elements do not provide the exact view
count, any uploader info (name, URL, avatar, verified status) and the upload
date.

All service's LinkHandlers are now using the singleton pattern and some code
has been also improved on the files changed.

Co-authored-by: ThetaDev <t.testboy@gmail.com>
Co-authored-by: Stypox <stypox@pm.me>
This commit is contained in:
AudricV 2023-07-14 23:46:48 +02:00 committed by Stypox
parent 4586067934
commit 7366eab156
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
15 changed files with 1498 additions and 379 deletions

View File

@ -0,0 +1,271 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Shared functions for extracting YouTube channel pages and tabs.
*/
public final class YoutubeChannelHelper {
private YoutubeChannelHelper() {
}
/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath a YouTube channel ID or URL path
* @return a YouTube channel ID
* @throws IOException if a channel resolve request failed
* @throws ExtractionException if a channel resolve request response could not be parsed or is
* invalid
*/
@Nonnull
public static String resolveChannelId(@Nonnull final String idOrPath)
throws ExtractionException, IOException {
final String[] channelId = idOrPath.split("/");
if (channelId[0].startsWith("UC")) {
return channelId[0];
}
// If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
// the InnerTube API to get the channel id.
// Otherwise, we couldn't get information about the channel associated with this URL, if
// there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"navigation/resolve_url", body, Localization.DEFAULT);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
return browseId;
}
}
return channelId[1];
}
/**
* Response data object for {@link #getChannelResponse(String, String, Localization,
* ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}).
*/
public static final class ChannelResponseData {
/**
* The channel response as a JSON object, after all redirects.
*/
@Nonnull
public final JsonObject jsonResponse;
/**
* The channel ID after all redirects.
*/
@Nonnull
public final String channelId;
private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
@Nonnull final String channelId) {
this.jsonResponse = jsonResponse;
this.channelId = channelId;
}
}
/**
* Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
*
* <p>
* Redirections to other channels such as are supported to up to 3 redirects, which could
* happen for instance for localized channels or auto-generated ones such as the {@code Movies
* and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
* and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
* {@code UClgRkhTL3_hImCAmdLfDE4g} one).
* </p>
*
* @param channelId a valid YouTube channel ID
* @param parameters the parameters to specify the YouTube channel tab; if invalid ones are
* specified, YouTube should return the {@code Home} tab
* @param localization the {@link Localization} to use
* @param country the {@link ContentCountry} to use
* @return a {@link ChannelResponseData channel response data}
* @throws IOException if a channel request failed
* @throws ExtractionException if a channel request response could not be parsed or is invalid
*/
@Nonnull
public static ChannelResponseData getChannelResponse(@Nonnull final String channelId,
@Nonnull final String parameters,
@Nonnull final Localization localization,
@Nonnull final ContentCountry country)
throws ExtractionException, IOException {
String id = channelId;
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, country)
.value("browseId", id)
.value("params", parameters)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"browse", body, localization);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint")
.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Got no channel response");
}
defaultAlertsCheck(ajaxJson);
return new ChannelResponseData(ajaxJson, id);
}
/**
* Assert that a channel JSON response does not contain an {@code error} JSON object.
*
* @param jsonResponse a channel JSON response
* @throws ContentNotAvailableException if the channel was not found
*/
private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
/**
* A channel header response.
*
* <p>
* This class allows the distinction between a classic header and a carousel one, used for
* auto-generated ones like the gaming or music topic channels and for big events such as the
* Coachella music festival, which have a different data structure and do not return the same
* properties.
* </p>
*/
public static final class ChannelHeader {
/**
* The channel header JSON response.
*/
@Nonnull
public final JsonObject json;
/**
* Whether the header is a {@code carouselHeaderRenderer}.
*
* <p>
* See the class documentation for more details.
* </p>
*/
public final boolean isCarouselHeader;
private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}
/**
* Get a channel header as an {@link Optional} it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject("header");
if (header.has("c4TabbedHeaderRenderer")) {
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (header.has("carouselHeaderRenderer")) {
return header.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.map(item -> item.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
}

View File

@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper {
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
return prepareDesktopJsonBuilder(localization, contentCountry, null);
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.end()
.value("platform", "DESKTOP");
if (visitorData != null) {
builder.value("visitorData", visitorData);
}
return builder.end()
.object("request")
.array("internalExperimentFlags")
.end()

View File

@ -8,6 +8,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService {
return YoutubeChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return YoutubeChannelTabLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return YoutubePlaylistLinkHandlerFactory.getInstance();
@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService {
return new YoutubeChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new YoutubeChannelTabExtractor(this, linkHandler);
}
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this);
final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance();
// add kiosks here e.g.:
try {
list.addKioskEntry(
(streamingService, url, id) -> new YoutubeTrendingExtractor(
YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
h.fromUrl(url),
id
),
new YoutubeTrendingLinkHandlerFactory(),
h,
YoutubeTrendingExtractor.KIOSK_ID
);
list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID);

View File

@ -1,39 +1,35 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -59,22 +55,24 @@ import javax.annotation.Nullable;
*/
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private Optional<JsonObject> channelHeader;
private boolean isCarouselHeader = false;
private JsonObject videoTab;
private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
private String channelId;
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it
* If a channel is age-restricted, its pages are only accessible to logged-in and
* age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
* the following metadata: channel name and channel avatar.
*
* <p>
* "Movies & Shows":
* <pre>
* UCuJcl0Ju-gPDoksRjK1ya-w
* UChBfWrfBXL9wS6tQtgjt_OQ UClgRkhTL3_hImCAmdLfDE4g
* UCok7UTQQEP1Rsctxiv3gwSQ
* </pre>
* This restriction doesn't seem to apply to all countries.
* </p>
*/
private String redirectedChannelId;
@Nullable
private JsonObject channelAgeGateRenderer;
public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelPath = super.getId();
final String[] channelId = channelPath.split("/");
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final String id = resolveChannelId(channelPath);
// Fetch Videos tab
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id,
"EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
}
} else {
id = channelId[1];
}
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
"");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Could not fetch initial JSON data");
}
initialData = ajaxJson;
YoutubeParsingHelper.defaultAlertsCheck(initialData);
jsonResponse = data.jsonResponse;
channelId = data.channelId;
channelAgeGateRenderer = getChannelAgeGateRenderer();
}
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
@Nonnull
private Optional<JsonObject> getChannelHeader() {
if (channelHeader == null) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
@Nullable
private JsonObject getChannelAgeGateRenderer() {
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
} else {
channelHeader = Optional.empty();
}
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
}
@Nonnull
private Optional<YoutubeChannelHelper.ChannelHeader> getChannelHeader() {
//noinspection OptionalAssignedToNull
if (channelHeader == null) {
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
}
return channelHeader;
}
@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getId() throws ParsingException {
return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
.or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
.or(() -> Optional.ofNullable(redirectedChannelId))
.orElseThrow(() -> new ParsingException("Could not get channel id"));
.or(() -> Optional.ofNullable(channelId))
.orElseThrow(() -> new ParsingException("Could not get channel ID"));
}
@Nonnull
@Override
public String getName() throws ParsingException {
final String mdName = initialData.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(mdName)) {
return mdName;
if (channelAgeGateRenderer != null) {
return channelAgeGateRenderer.getString("channelTitle");
}
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Object title = header.get().get("title");
final String metadataRendererTitle = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataRendererTitle)) {
return metadataRendererTitle;
}
return getChannelHeader().flatMap(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
return Optional.of(headerName);
}
}
}
throw new ParsingException("Could not get channel name");
return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Override
public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url")
))
.map(YoutubeParsingHelper::fixThumbnailUrl)
.orElseThrow(() -> new ParsingException("Could not get avatar"));
final JsonObject avatarJsonObjectContainer;
if (channelAgeGateRenderer != null) {
avatarJsonObjectContainer = channelAgeGateRenderer;
} else {
avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
}
return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url"));
}
@Override
public String getBannerUrl() throws ParsingException {
if (channelAgeGateRenderer != null) {
return "";
}
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails")
.getObject(0).getString("url")
))
header.json.getObject("banner")
.getArray("thumbnails")
.getObject(0)
.getString("url")))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no banner is
@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
// RSS feeds are accessible for age-restricted channels, no need to check whether a channel
// has a channelAgeGateRenderer
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (final Exception e) {
@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
if (channelAgeGateRenderer != null) {
return UNKNOWN_SUBSCRIBER_COUNT;
}
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null;
if (header.get().has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText");
} else if (header.get().has("subtitle")) {
textObject = header.get().getObject("subtitle");
if (header.has("subscriberCountText")) {
textObject = header.getObject("subscriberCountText");
} else if (header.has("subtitle")) {
textObject = header.getObject("subtitle");
}
if (textObject != null) {
@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
}
return UNKNOWN_SUBSCRIBER_COUNT;
}
@Override
public String getDescription() throws ParsingException {
if (channelAgeGateRenderer != null) {
return null;
}
try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer")
return jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("description");
} catch (final Exception e) {
throw new ParsingException("Could not get channel description", e);
@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (isCarouselHeader) {
return true;
if (channelAgeGateRenderer != null) {
return false;
}
return getChannelHeader()
.map(header -> header.getArray("badges"))
.map(YoutubeParsingHelper::isVerified)
.orElse(false);
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
if (getVideoTab() != null) {
final JsonObject tabContent = getVideoTab().getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
public List<ListLinkHandler> getTabs() throws ParsingException {
if (channelAgeGateRenderer == null) {
return getTabsForNonAgeRestrictedChannels();
}
return new InfoItemsPage<>(collector, nextPage);
return getTabsForAgeRestrictedChannels();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
*/
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (videoTab != null) {
return videoTab;
}
final JsonArray tabs = initialData.getObject("contents")
@Nonnull
private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
final JsonArray responseTabs = jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
final JsonObject foundVideoTab = tabs.stream()
.filter(Objects::nonNull)
final List<ListLinkHandler> tabs = new ArrayList<>();
final Consumer<String> addNonVideosTab = tabName -> {
try {
tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
channelId, List.of(tabName), ""));
} catch (final ParsingException ignored) {
// Do not add the tab if we couldn't create the LinkHandler
}
};
final String name = getName();
final String url = getUrl();
final String id = getId();
responseTabs.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer")
&& tab.getObject("tabRenderer")
.getString("title", "")
.equals("Videos"))
.findFirst()
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.orElseThrow(
() -> new ContentNotSupportedException("This channel has no Videos tab"));
.forEach(tabRenderer -> {
final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null) {
final String[] urlParts = tabUrl.split("/");
if (urlParts.length == 0) {
return;
}
final String messageRendererText = getTextFromObject(
foundVideoTab.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("messageRenderer")
.getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
final String urlSuffix = urlParts[urlParts.length - 1];
switch (urlSuffix) {
case "videos":
// Since the Videos tab has already its contents fetched, make
// sure it is in the first position
// YoutubeChannelTabExtractor still supports fetching this tab
tabs.add(0, new ReadyChannelTabListLinkHandler(
tabUrl,
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);
break;
case "streams":
addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
break;
case "playlists":
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
break;
case "channels":
addNonVideosTab.accept(ChannelTabs.CHANNELS);
break;
}
}
});
return Collections.unmodifiableList(tabs);
}
@Nonnull
private List<ListLinkHandler> getTabsForAgeRestrictedChannels() throws ParsingException {
// As we don't have access to the channel tabs list, consider that the channel has videos,
// shorts and livestreams, the data only accessible without login on YouTube's desktop
// client using uploads system playlists
// The playlists channel tab is still available on YouTube Music, but this is not
// implemented in the extractor
final List<ListLinkHandler> tabs = new ArrayList<>();
final String channelUrl = getUrl();
final Consumer<String> addTab = tabName ->
tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName,
channelId, tabName, YoutubeChannelTabPlaylistExtractor::new));
addTab.accept(ChannelTabs.VIDEOS);
addTab.accept(ChannelTabs.SHORTS);
addTab.accept(ChannelTabs.LIVESTREAMS);
return Collections.unmodifiableList(tabs);
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
if (channelAgeGateRenderer != null) {
return List.of();
}
videoTab = foundVideoTab;
return foundVideoTab;
return jsonResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getArray("tags")
.stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.collect(Collectors.toUnmodifiableList());
}
}

View File

@ -0,0 +1,486 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} implementation for the YouTube service.
*
* <p>
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
* {@code Channels} tabs.
* </p>
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
/**
* Whether the visitor data extracted from the initial channel response is required to be used
* for continuations.
*
* <p>
* A valid {@code visitorData} is required to get continuations of shorts in channels.
* </p>
*
* <p>
* It should be not used when it is not needed, in order to reduce YouTube's tracking.
* </p>
*/
private final boolean useVisitorData;
private JsonObject jsonResponse;
private String channelId;
@Nullable
private String visitorData;
public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
useVisitorData = getName().equals(ChannelTabs.SHORTS);
}
@Nonnull
private String getChannelTabsParameters() throws ParsingException {
final String name = getName();
switch (name) {
case ChannelTabs.VIDEOS:
return "EgZ2aWRlb3PyBgQKAjoA";
case ChannelTabs.SHORTS:
return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
case ChannelTabs.LIVESTREAMS:
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
case ChannelTabs.PLAYLISTS:
return "EglwbGF5bGlzdHPyBgQKAkIA";
case ChannelTabs.CHANNELS:
return "EghjaGFubmVsc_IGBAoCUgA%3D";
}
throw new ParsingException("Unsupported channel tab: " + name);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
channelId = resolveChannelId(super.getId());
final String params = getChannelTabsParameters();
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
params, getExtractorLocalization(), getExtractorContentCountry());
jsonResponse = data.jsonResponse;
channelId = data.channelId;
if (useVisitorData) {
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
}
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
try {
return YoutubeChannelTabLinkHandlerFactory.getInstance()
.getUrl("channel/" + getId(), List.of(getName()), "");
} catch (final ParsingException e) {
return super.getUrl();
}
}
@Nonnull
@Override
public String getId() throws ParsingException {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (!id.isEmpty()) {
return id;
}
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
}
if (!isNullOrEmpty(channelId)) {
return channelId;
} else {
throw new ParsingException("Could not get channel ID");
}
}
protected String getChannelName() {
final String metadataName = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataName)) {
return metadataName;
}
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
})
.orElse("");
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
JsonArray items = new JsonArray();
final Optional<JsonObject> tab = getTabData();
if (tab.isPresent()) {
final JsonObject tabContent = tab.get().getObject("content");
items = tabContent.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("gridRenderer")
.getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer")
.getArray("contents");
if (items.isEmpty()) {
items = tabContent.getObject("sectionListRenderer")
.getArray("contents");
}
}
}
// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
// We also need to set the visitor data here when it should be enabled, as it is required
// to get continuations on some channel tabs, and we need a way to pass it between pages
final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
? List.of(getChannelName(), getUrl(), visitorData)
: List.of(getChannelName(), getUrl());
final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
.orElse(null);
final Page nextPage = getNextPageFrom(continuation, channelIds);
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
.map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
.findFirst()
.orElse(new JsonObject());
final JsonObject continuation = collectItemsFrom(collector,
sectionListContinuation.getArray("continuationItems"), channelIds)
.orElse(null);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
Optional<JsonObject> getTabData() {
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url", "").endsWith(urlSuffix))
.findFirst()
// Check if tab has no content
.filter(tabRenderer -> {
final JsonArray tabContents = tabRenderer.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents");
return tabContents.size() != 1
|| !tabContents.getObject(0).has("messageRenderer");
});
}
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final List<String> channelIds) {
return items.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(item -> collectItem(collector, item, channelIds))
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
}
private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject item,
@Nonnull final List<String> channelIds) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
if (item.has("richItemRenderer")) {
final JsonObject richItem = item.getObject("richItemRenderer")
.getObject("content");
if (richItem.has("videoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("videoRenderer"));
} else if (richItem.has("reelItemRenderer")) {
getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("reelItemRenderer"));
} else if (richItem.has("playlistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("playlistRenderer"));
}
} else if (item.has("gridVideoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
item.getObject("gridVideoRenderer"));
} else if (item.has("gridPlaylistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("gridPlaylistRenderer"));
} else if (item.has("gridChannelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("gridChannelRenderer")));
} else if (item.has("shelfRenderer")) {
return collectItem(collector, item.getObject("shelfRenderer")
.getObject("content"), channelIds);
} else if (item.has("itemSectionRenderer")) {
return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
.getArray("contents"), channelIds);
} else if (item.has("horizontalListRenderer")) {
return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
.getArray("items"), channelIds);
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelIds);
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
return Optional.empty();
}
@Nonnull
private Consumer<JsonObject> getCommitVideoConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitReelItemConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return reelItemRenderer -> collector.commit(
new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitPlaylistConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final List<String> channelIds) {
return playlistRenderer -> collector.commit(
new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry(),
useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already
* fetched.
*/
public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
private final JsonObject tabRenderer;
private final String channelName;
private final String channelId;
private final String channelUrl;
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
final String channelName,
final String channelId,
final String channelUrl) {
super(service, linkHandler);
this.tabRenderer = tabRenderer;
this.channelName = channelName;
this.channelId = channelId;
this.channelUrl = channelUrl;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// Nothing to do, the initial data was already fetched and is stored in the link handler
}
@Nonnull
@Override
public String getId() throws ParsingException {
return channelId;
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return channelUrl;
}
@Override
protected String getChannelName() {
return channelName;
}
@Override
Optional<JsonObject> getTabData() {
return Optional.of(tabRenderer);
}
}
}

View File

@ -0,0 +1,191 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} for YouTube system playlists using a
* {@link YoutubePlaylistExtractor} instance.
*
* <p>
* It is currently used to bypass age-restrictions on channels marked as age-restricted by their
* owner(s).
* </p>
*/
public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor {
private final PlaylistExtractor playlistExtractorInstance;
private boolean playlistExisting;
/**
* Construct a {@link YoutubeChannelTabPlaylistExtractor} instance.
*
* @param service a {@link StreamingService} implementation, which must be the YouTube
* one
* @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting
* with `UC`) and one of the given and supported content filters:
* {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS},
* {@link ChannelTabs#LIVESTREAMS}
* @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the
* required arguments
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
super(service, linkHandler);
final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler);
this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
playlistExtractorInstance.onFetchPage(downloader);
if (!playlistExisting) {
playlistExisting = true;
}
} catch (final ContentNotAvailableException e) {
// If a channel has no content of the type requested, the corresponding system playlist
// won't exist, so a ContentNotAvailableException would be thrown
// Ignore such issues in this case
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistInitialPage =
playlistExtractorInstance.getInitialPage();
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistInitialPage.getItems());
return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
playlistInitialPage.getErrors());
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistPage = playlistExtractorInstance.getPage(page);
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistPage.getItems());
return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
playlistPage.getErrors());
}
/**
* Get a playlist {@link ListLinkHandler} from a channel tab one.
*
* <p>
* This method converts a channel ID without its {@code UC} prefix into a YouTube system
* playlist, depending on the first content filter provided in the given
* {@link ListLinkHandler}.
* </p>
*
* <p>
* The first content filter must be a channel tabs one among the
* {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and
* {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists
* with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the
* {@code UC} part is appended.
* </p>
*
* @param originalLinkHandler the original {@link ListLinkHandler} with which a
* {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed
*
* @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance
* needed to extract channel tabs data from a system playlist
* @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the
* required criteria above
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
@Nonnull
private ListLinkHandler getPlaylistLinkHandler(
@Nonnull final ListLinkHandler originalLinkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
final List<String> contentFilters = originalLinkHandler.getContentFilters();
if (contentFilters.isEmpty()) {
throw new IllegalArgumentException("A content filter is required");
}
final String channelId = originalLinkHandler.getId();
if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) {
throw new IllegalArgumentException("Invalid channel ID");
}
final String channelIdWithoutUc = channelId.substring(2);
final String playlistId;
switch (contentFilters.get(0)) {
case ChannelTabs.VIDEOS:
playlistId = "UULF" + channelIdWithoutUc;
break;
case ChannelTabs.SHORTS:
playlistId = "UUSH" + channelIdWithoutUc;
break;
case ChannelTabs.LIVESTREAMS:
playlistId = "UULV" + channelIdWithoutUc;
break;
default:
throw new IllegalArgumentException(
"Only Videos, Shorts and Livestreams tabs can extracted as playlists");
}
try {
final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance()
.getUrl(playlistId);
return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), "");
} catch (final ParsingException e) {
// This should be not reachable, as the given playlist ID should be valid and
// YoutubePlaylistLinkHandlerFactory doesn't throw any exception
throw new SystemPlaylistUrlCreationException(
"Could not create a YouTube playlist from a valid playlist ID", e);
}
}
/**
* Exception thrown when a YouTube system playlist URL could not be created.
*
* <p>
* This exception should be never thrown, as given playlist IDs should be always valid.
* </p>
*/
public static final class SystemPlaylistUrlCreationException extends RuntimeException {
SystemPlaylistUrlCreationException(final String message, final Throwable cause) {
super(message, cause);
}
}
}

View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url");
JsonArray thumbnails = playlistInfoItem.getArray("thumbnails")
.getObject(0)
.getArray("thumbnails");
if (thumbnails.isEmpty()) {
thumbnails = playlistInfoItem.getObject("thumbnail")
.getArray("thumbnails");
}
return fixThumbnailUrl(url);
return fixThumbnailUrl(thumbnails.getObject(0).getString("url"));
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public long getStreamCount() throws ParsingException {
String videoCountText = playlistInfoItem.getString("videoCount");
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText"));
}
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
}
if (videoCountText == null) {
throw new ParsingException("Could not get stream count");
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
playlistInfoItem.getString("videoCount")));
return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}

View File

@ -0,0 +1,147 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
*
* <p>
* {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
* place and every major client. They provide a limited amount of information and do not provide
* the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
* </p>
*/
public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
@Nonnull
private final JsonObject reelInfo;
@Nullable
private final TimeAgoParser timeAgoParser;
public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
@Nullable final TimeAgoParser timeAgoParser) {
this.reelInfo = reelInfo;
this.timeAgoParser = timeAgoParser;
}
@Override
public String getName() throws ParsingException {
return getTextFromObject(reelInfo.getObject("headline"));
}
@Override
public String getUrl() throws ParsingException {
try {
final String videoId = reelInfo.getString("videoId");
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
} catch (final Exception e) {
throw new ParsingException("Could not get URL", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
return getThumbnailUrlFromInfoItem(reelInfo);
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
// Duration of reelItems is only provided in the accessibility data
// example: "VIDEO TITLE - 49 seconds - play video"
// "VIDEO TITLE - 1 minute, 1 second - play video"
final String accessibilityLabel = reelInfo.getObject("accessibility")
.getObject("accessibilityData").getString("label");
if (accessibilityLabel == null || timeAgoParser == null) {
return 0;
}
// This approach may be language dependent
final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
if (labelParts.length > 2) {
final String textualDuration = labelParts[labelParts.length - 2];
return timeAgoParser.parseDuration(textualDuration);
}
return -1;
}
@Override
public long getViewCount() throws ParsingException {
final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
if (!isNullOrEmpty(viewCountText)) {
// This approach is language dependent
if (viewCountText.toLowerCase().contains("no views")) {
return 0;
}
return Utils.mixedNumberWordToLong(viewCountText);
}
throw new ParsingException("Could not get short view count");
}
@Override
public boolean isShortFormContent() throws ParsingException {
return true;
}
// All the following properties cannot be obtained from reelItemRenderers
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return null;
}
@Override
public String getUploaderUrl() throws ParsingException {
return null;
}
@Nullable
@Override
public String getUploaderAvatarUrl() throws ParsingException {
return null;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return null;
}
}

View File

@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
@Override
public String getUrl(final String id,
final List<String> contentFilters,
final String searchFilter) {
final String searchFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id;
}
@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();

View File

@ -0,0 +1,73 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
new YoutubeChannelTabLinkHandlerFactory();
private YoutubeChannelTabLinkHandlerFactory() {
}
public static YoutubeChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab)
throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.SHORTS:
return "/shorts";
case ChannelTabs.LIVESTREAMS:
return "/streams";
case ChannelTabs.PLAYLISTS:
return "/playlists";
case ChannelTabs.CHANNELS:
return "/channels";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0));
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return YoutubeChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
try {
getId(url);
} catch (final ParsingException e) {
return false;
}
return true;
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.SHORTS,
ChannelTabs.LIVESTREAMS,
ChannelTabs.PLAYLISTS,
ChannelTabs.CHANNELS
};
}
}

View File

@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
}
@Override
public String getUrl(final String id) {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
// we need the same id, avoids duplicate code
public String getId(final String urlString)
throws ParsingException, UnsupportedOperationException {
// We need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
}
@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id);
}
}

View File

@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id, final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/playlist?list=" + id;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);

View File

@ -13,6 +13,9 @@ import javax.annotation.Nonnull;
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
new YoutubeSearchQueryHandlerFactory();
public static final String ALL = "all";
public static final String VIDEOS = "videos";
public static final String CHANNELS = "channels";
@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
@Nonnull
public static YoutubeSearchQueryHandlerFactory getInstance() {
return new YoutubeSearchQueryHandlerFactory();
return INSTANCE;
}
@Override
public String getUrl(final String searchString,
@Nonnull final List<String> contentFilters,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
if (!contentFilters.isEmpty()) {
final String contentFilter = contentFilters.get(0);
switch (contentFilter) {
case ALL:
default:
break;
case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:

View File

@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
public String getUrl(final String id) {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
public String getId(final String theUrlString)
throws ParsingException, IllegalArgumentException {
throws ParsingException, UnsupportedOperationException {
String urlString = theUrlString;
try {
final URI uri = new URI(urlString);

View File

@ -1,28 +1,29 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe.
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor.
*
* NewPipe is free software: you can redistribute it and/or modify
* NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* NewPipe Extractor is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
@ -30,16 +31,27 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeTrendingLinkHandlerFactory INSTANCE =
new YoutubeTrendingLinkHandlerFactory();
private YoutubeTrendingLinkHandlerFactory() {
}
public static YoutubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE;
}
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/feed/trending";
}
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "Trending";
}