From d8ce08d9694b53d105a0fd8d9ad19c26b3de84bd Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 12 Nov 2022 09:31:05 +0530 Subject: [PATCH] Use immutable Map factory methods. --- .../PeertubeTrendingLinkHandlerFactory.java | 31 ++--- .../soundcloud/SoundcloudParsingHelper.java | 3 +- .../youtube/YoutubeParsingHelper.java | 122 +++++++++--------- .../YoutubeDashManifestCreatorsUtils.java | 45 +++---- .../YoutubeMixPlaylistExtractor.java | 8 +- .../extractors/YoutubePlaylistExtractor.java | 6 +- 6 files changed, 100 insertions(+), 115 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index e90293ae6..bded13ff2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -4,8 +4,6 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,27 +12,16 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac private static final PeertubeTrendingLinkHandlerFactory INSTANCE = new PeertubeTrendingLinkHandlerFactory(); - public static final Map KIOSK_MAP; - public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; public static final String KIOSK_MOST_LIKED = "Most liked"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; - static { - final Map map = new HashMap<>(); - map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); - map.put(KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes"); - map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); - map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); - KIOSK_MAP = Collections.unmodifiableMap(map); - - final Map reverseMap = new HashMap<>(); - for (final Map.Entry entry : KIOSK_MAP.entrySet()) { - reverseMap.put(entry.getValue(), entry.getKey()); - } - REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); - } + public static final Map KIOSK_MAP = Map.of( + KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending", + KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes", + KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt", + KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); public static PeertubeTrendingLinkHandlerFactory getInstance() { return INSTANCE; @@ -66,10 +53,12 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac return KIOSK_RECENT; } else if (cleanUrl.contains("/videos/local")) { return KIOSK_LOCAL; - } else if (REVERSE_KIOSK_MAP.containsKey(cleanUrl)) { - return REVERSE_KIOSK_MAP.get(cleanUrl); } else { - throw new ParsingException("no id found for this url"); + return KIOSK_MAP.entrySet().stream() + .filter(entry -> cleanUrl.equals(entry.getValue())) + .findFirst() + .map(Map.Entry::getKey) + .orElseThrow(() -> new ParsingException("no id found for this url")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 57deb64a2..a0bd239df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -64,8 +64,7 @@ public final class SoundcloudParsingHelper { // The one containing the client id will likely be the last one Collections.reverse(possibleScripts); - final Map> headers = Collections.singletonMap("Range", - Collections.singletonList("bytes=0-50000")); + final var headers = Map.of("Range", List.of("bytes=0-50000")); for (final Element element : possibleScripts) { final String srcUrl = element.attr("src"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index e65361c8b..b8d37705e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -25,7 +25,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static java.util.Collections.singletonList; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; @@ -61,7 +60,6 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -92,6 +90,11 @@ public final class YoutubeParsingHelper { public static final String YOUTUBEI_V1_GAPIS_URL = "https://youtubei.googleapis.com/youtubei/v1/"; + /** + * The base URL of YouTube Music. + */ + private static final String YOUTUBE_MUSIC_URL = "https://music.youtube.com"; + /** * A parameter to disable pretty-printed response of InnerTube requests, to reduce response * sizes. @@ -554,9 +557,7 @@ public final class YoutubeParsingHelper { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", singletonList("1")); - headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION)); + final var headers = getClientHeaders("1", HARDCODED_CLIENT_VERSION); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -578,9 +579,7 @@ public final class YoutubeParsingHelper { return; } final String url = "https://www.youtube.com/sw.js"; - final Map> headers = new HashMap<>(); - headers.put("Origin", singletonList("https://www.youtube.com")); - headers.put("Referer", singletonList("https://www.youtube.com")); + final var headers = getOriginReferrerHeaders("https://www.youtube.com"); final String response = getDownloader().get(url, headers).responseBody(); try { clientVersion = getStringResultFromRegexArray(response, @@ -799,11 +798,9 @@ public final class YoutubeParsingHelper { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1])); - headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2])); - headers.put("Origin", singletonList("https://music.youtube.com")); - headers.put("Referer", singletonList("https://music.youtube.com")); + final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); + headers.putAll(getClientHeaders(HARDCODED_YOUTUBE_MUSIC_KEY[1], + HARDCODED_YOUTUBE_MUSIC_KEY[2])); final Response response = getDownloader().postWithContentTypeJson(url, headers, json); // Ensure to have a valid response @@ -826,14 +823,12 @@ public final class YoutubeParsingHelper { try { final String url = "https://music.youtube.com/sw.js"; - final Map> headers = new HashMap<>(); - headers.put("Origin", singletonList("https://music.youtube.com")); - headers.put("Referer", singletonList("https://music.youtube.com")); + final var headers = getOriginReferrerHeaders("https://music.youtube.com"); final String response = getDownloader().get(url, headers).responseBody(); - musicClientVersion = getStringResultFromRegexArray(response, - INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); - musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response); + musicClientVersion = getStringResultFromRegexArray(response, + INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); + musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); + musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response); } catch (final Exception e) { final String url = "https://music.youtube.com/?ucbcb=1"; final String html = getDownloader().get(url, getCookieHeader()).responseBody(); @@ -1176,8 +1171,7 @@ public final class YoutubeParsingHelper { final byte[] body, final Localization localization) throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - addYouTubeHeaders(headers); + final var headers = getYouTubeHeaders(); return JsonUtils.toJsonObject(getValidJsonResponseBody( getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key=" @@ -1209,9 +1203,8 @@ public final class YoutubeParsingHelper { @Nonnull final String userAgent, @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - headers.put("User-Agent", singletonList(userAgent)); - headers.put("X-Goog-Api-Format-Version", singletonList("2")); + final var headers = Map.of("User-Agent", List.of(userAgent), + "X-Goog-Api-Format-Version", List.of("2")); final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + DISABLE_PRETTY_PRINT_PARAMETER; @@ -1423,40 +1416,59 @@ public final class YoutubeParsingHelper { + ")"; } + /** + * Returns a {@link Map} containing the required YouTube Music headers. + */ @Nonnull public static Map> getYoutubeMusicHeaders() { - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("https://music.youtube.com")); + final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); + headers.putAll(getClientHeaders(youtubeMusicKey[1], youtubeMusicKey[2])); return headers; } /** - * Add required headers and cookies to an existing headers Map. - * @see #addClientInfoHeaders(Map) - * @see #addCookieHeader(Map) + * Returns a {@link Map} containing the required YouTube headers. */ - public static void addYouTubeHeaders(final Map> headers) - throws IOException, ExtractionException { - addClientInfoHeaders(headers); - addCookieHeader(headers); + public static Map> getYouTubeHeaders() + throws ExtractionException, IOException { + final var headers = getClientInfoHeaders(); + headers.put("Cookie", List.of(generateConsentCookie())); + return headers; } /** - * Add the X-YouTube-Client-Name, X-YouTube-Client-Version, - * Origin, and Referer headers. - * @param headers The headers which should be completed + * Returns a {@link Map} containing the {@code X-YouTube-Client-Name}, + * {@code X-YouTube-Client-Version}, {@code Origin}, and {@code Referer} headers. */ - public static void addClientInfoHeaders(@Nonnull final Map> headers) - throws IOException, ExtractionException { - headers.computeIfAbsent("Origin", k -> singletonList("https://www.youtube.com")); - headers.computeIfAbsent("Referer", k -> singletonList("https://www.youtube.com")); - headers.computeIfAbsent("X-YouTube-Client-Name", k -> singletonList("1")); - if (headers.get("X-YouTube-Client-Version") == null) { - headers.put("X-YouTube-Client-Version", singletonList(getClientVersion())); - } + public static Map> getClientInfoHeaders() + throws ExtractionException, IOException { + final var headers = new HashMap<>(getOriginReferrerHeaders("https://www.youtube.com")); + headers.putAll(getClientHeaders("1", getClientVersion())); + return headers; + } + + /** + * Returns an unmodifiable {@link Map} containing the {@code Origin} and {@code Referer} + * headers set to the given URL. + * + * @param url The URL to be set as the origin and referrer. + */ + private static Map> getOriginReferrerHeaders(@Nonnull final String url) { + final var urlList = List.of(url); + return Map.of("Origin", urlList, "Referer", urlList); + } + + /** + * Returns an unmodifiable {@link Map} containing the {@code X-YouTube-Client-Name} and + * {@code X-YouTube-Client-Version} headers. + * + * @param name The X-YouTube-Client-Name value. + * @param version X-YouTube-Client-Version value. + */ + private static Map> getClientHeaders(@Nonnull final String name, + @Nonnull final String version) { + return Map.of("X-YouTube-Client-Name", List.of(name), + "X-YouTube-Client-Version", List.of(version)); } /** @@ -1464,19 +1476,7 @@ public final class YoutubeParsingHelper { * @return A singleton map containing the header. */ public static Map> getCookieHeader() { - return Collections.singletonMap("Cookie", singletonList(generateConsentCookie())); - } - - /** - * Add the CONSENT cookie to prevent redirect to consent.youtube.com - * @param headers the headers which should be completed - */ - public static void addCookieHeader(@Nonnull final Map> headers) { - if (headers.get("Cookie") == null) { - headers.put("Cookie", Collections.singletonList(generateConsentCookie())); - } else { - headers.get("Cookie").add(generateConsentCookie()); - } + return Map.of("Cookie", List.of(generateConsentCookie())); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 46bd32420..d7a385ac4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -1,5 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientInfoHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -13,6 +22,14 @@ import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + import javax.annotation.Nonnull; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; @@ -25,24 +42,6 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import java.io.IOException; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - /** * Utilities and constants for YouTube DASH manifest creators. * @@ -583,9 +582,9 @@ public final class YoutubeDashManifestCreatorsUtils { } } else if (isAndroidStreamingUrl || isIosStreamingUrl) { try { - final Map> headers = Collections.singletonMap("User-Agent", - Collections.singletonList(isAndroidStreamingUrl - ? getAndroidUserAgent(null) : getIosUserAgent(null))); + final var headers = Map.of("User-Agent", + List.of(isAndroidStreamingUrl ? getAndroidUserAgent(null) + : getIosUserAgent(null))); final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8); return downloader.post(baseStreamingUrl, headers, emptyBody); } catch (final IOException | ExtractionException e) { @@ -705,9 +704,7 @@ public final class YoutubeDashManifestCreatorsUtils { @Nonnull final String responseMimeTypeExpected) throws CreationException { try { - final Map> headers = new HashMap<>(); - headers.put("Origin", Collections.singletonList("https://www.youtube.com")); - headers.put("Referer", Collections.singletonList("https://www.youtube.com")); + final var headers = getClientInfoHeaders(); String responseMimeType = ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 02d9ed5aa..fb574e294 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -88,9 +88,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8); - final Map> headers = new HashMap<>(); // Cookie is required due to consent - addYouTubeHeaders(headers); + final var headers = getYouTubeHeaders(); final Response response = getDownloader().postWithContentTypeJson( YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, @@ -222,9 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); // Cookie is required due to consent - addYouTubeHeaders(headers); + final var headers = getYouTubeHeaders(); final Response response = getDownloader().postWithContentTypeJson(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index e9c5c0be7..38f1dabbb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -14,6 +14,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -29,11 +30,12 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public class YoutubePlaylistExtractor extends PlaylistExtractor { // Names of some objects in JSON response frequently used in this class private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";