[YouTube] Apply changes in Extractors except YoutubeMusicSearchExtractor

Also improve a bit some code related to the changes.

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -1,3 +1,23 @@
+/*
+ * Created by Christian Schabesberger on 25.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * YoutubeChannelExtractor.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
@@ -8,6 +28,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 
+import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
@@ -36,26 +57,6 @@ import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-/*
- * Created by Christian Schabesberger on 25.07.16.
- *
- * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
- * YoutubeChannelExtractor.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 public class YoutubeChannelExtractor extends ChannelExtractor {
 
     private JsonObject jsonResponse;
@@ -190,16 +191,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         .orElseThrow(() -> new ParsingException("Could not get channel name"));
     }
 
+    @Nonnull
     @Override
-    public String getAvatarUrl() throws ParsingException {
+    public List<Image> getAvatars() throws ParsingException {
         assertPageFetched();
         if (channelAgeGateRenderer != null) {
             return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
-                    .getArray("thumbnails")
+                    .getArray("thumbnails"))
-                    .getObject(0)
+                    .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
-                    .getString("url"))
+                    .orElseThrow(() -> new ParsingException("Could not get avatars"));
-                    .map(YoutubeParsingHelper::fixThumbnailUrl)
-                    .orElseThrow(() -> new ParsingException("Could not get avatar URL"));
         }
 
         return channelHeader.map(header -> {
@@ -210,56 +210,37 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                             .getObject("image")
                             .getObject("contentPreviewImageViewModel")
                             .getObject("image")
-                            .getArray("sources")
+                            .getArray("sources");
-                            .getObject(0)
-                            .getString("url");
 
                 case INTERACTIVE_TABBED:
                     return header.json.getObject("boxArt")
-                            .getArray("thumbnails")
+                            .getArray("thumbnails");
-                            .getObject(0)
-                            .getString("url");
 
                 case C4_TABBED:
                 case CAROUSEL:
                 default:
                     return header.json.getObject("avatar")
-                            .getArray("thumbnails")
+                            .getArray("thumbnails");
-                            .getObject(0)
-                            .getString("url");
             }
         })
-                .map(YoutubeParsingHelper::fixThumbnailUrl)
+                .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
-                .orElseThrow(() -> new ParsingException("Could not get avatar URL"));
+                .orElseThrow(() -> new ParsingException("Could not get avatars"));
     }
 
+    @Nonnull
     @Override
-    public String getBannerUrl() throws ParsingException {
+    public List<Image> getBanners() {
         assertPageFetched();
         if (channelAgeGateRenderer != null) {
-            return null;
+            return List.of();
         }
 
-        if (channelHeader.isPresent()) {
+        // No banner is available on pageHeaderRenderer headers
-            final ChannelHeader header = channelHeader.get();
+        return channelHeader.filter(header -> header.headerType != HeaderType.PAGE)
-            if (header.headerType == HeaderType.PAGE) {
+                .map(header -> header.json.getObject("banner")
-                // No banner is available on pageHeaderRenderer headers
+                        .getArray("thumbnails"))
-                return null;
+                .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
-            }
+                .orElse(List.of());
-
-            return Optional.ofNullable(header.json.getObject("banner")
-                    .getArray("thumbnails")
-                    .getObject(0)
-                    .getString("url"))
-                    .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
-                    .map(YoutubeParsingHelper::fixThumbnailUrl)
-                    // Channels may not have a banner, so no exception should be thrown if no
-                    // banner is found
-                    // Return null in this case
-                    .orElse(null);
-        }
-
-        return null;
     }
 
     @Override
@@ -359,9 +340,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         return "";
     }
 
+    @Nonnull
     @Override
-    public String getParentChannelAvatarUrl() {
+    public List<Image> getParentChannelAvatars() {
-        return "";
+        return List.of();
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -17,6 +17,8 @@ import com.grack.nanojson.JsonBuilder;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
 
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.Image.ResolutionLevel;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -34,6 +36,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.ImageSuffix;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
@@ -43,6 +46,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -53,6 +57,12 @@ import javax.annotation.Nullable;
  * {@code youtube.com/watch?v=videoId&list=playlistId}
  */
 public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
+    private static final List<ImageSuffix> IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of(
+            // sqdefault and maxresdefault image resolutions are not available on all
+            // videos, so don't add them in the list of available resolutions
+            new ImageSuffix("default.jpg", 90, 120, ResolutionLevel.LOW),
+            new ImageSuffix("mqdefault.jpg", 180, 320, ResolutionLevel.MEDIUM),
+            new ImageSuffix("hqdefault.jpg", 360, 480, ResolutionLevel.MEDIUM));
 
     /**
      * YouTube identifies mixes based on this cookie. With this information it can generate
@@ -126,18 +136,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
 
     @Nonnull
     @Override
-    public String getThumbnailUrl() throws ParsingException {
+    public List<Image> getThumbnails() throws ParsingException {
         try {
-            return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
+            return getThumbnailsFromPlaylistId(playlistData.getString("playlistId"));
         } catch (final Exception e) {
             try {
-                // Fallback to thumbnail of current video. Always the case for channel mix
+                // Fallback to thumbnail of current video. Always the case for channel mixes
-                return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
+                return getThumbnailsFromVideoId(initialData.getObject("currentVideoEndpoint")
                         .getObject("watchEndpoint").getString("videoId"));
             } catch (final Exception ignored) {
             }
 
-            throw new ParsingException("Could not get playlist thumbnail", e);
+            throw new ParsingException("Could not get playlist thumbnails", e);
         }
     }
 
@@ -153,10 +163,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         return "YouTube";
     }
 
+    @Nonnull
     @Override
-    public String getUploaderAvatarUrl() {
+    public List<Image> getUploaderAvatars() {
         // YouTube mixes are auto-generated by YouTube
-        return "";
+        return List.of();
     }
 
     @Override
@@ -264,14 +275,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
     }
 
     @Nonnull
-    private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId)
+    private List<Image> getThumbnailsFromPlaylistId(@Nonnull final String playlistId)
             throws ParsingException {
-        return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId));
+        return getThumbnailsFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId));
     }
 
     @Nonnull
-    private String getThumbnailUrlFromVideoId(final String videoId) {
+    private List<Image> getThumbnailsFromVideoId(@Nonnull final String videoId) {
-        return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
+        final String baseUrl = "https://i.ytimg.com/vi/" + videoId + "/";
+        return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream()
+                .map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(),
+                        imageSuffix.getHeight(), imageSuffix.getWidth(),
+                        imageSuffix.getResolutionLevel()))
+                .collect(Collectors.toUnmodifiableList());
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -3,10 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -15,6 +15,7 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
 
+import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -33,6 +34,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -160,39 +162,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Nonnull
     @Override
-    public String getThumbnailUrl() throws ParsingException {
+    public List<Image> getThumbnails() throws ParsingException {
-        String url;
+        final JsonArray playlistMetadataThumbnailsArray;
         if (isNewPlaylistInterface) {
-            url = getPlaylistHeader().getObject("playlistHeaderBanner")
+            playlistMetadataThumbnailsArray = getPlaylistHeader().getObject("playlistHeaderBanner")
                     .getObject("heroPlaylistThumbnailRenderer")
                     .getObject("thumbnail")
-                    .getArray("thumbnails")
+                    .getArray("thumbnails");
-                    .getObject(0)
-                    .getString("url");
         } else {
-            url = getPlaylistInfo().getObject("thumbnailRenderer")
+            playlistMetadataThumbnailsArray = playlistInfo.getObject("thumbnailRenderer")
                     .getObject("playlistVideoThumbnailRenderer")
                     .getObject("thumbnail")
-                    .getArray("thumbnails")
+                    .getArray("thumbnails");
-                    .getObject(0)
+        }
-                    .getString("url");
+
+        if (!isNullOrEmpty(playlistMetadataThumbnailsArray)) {
+            return getImagesFromThumbnailsArray(playlistMetadataThumbnailsArray);
         }
 
         // This data structure is returned in both layouts
-        if (isNullOrEmpty(url)) {
+        final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
-            url = browseResponse.getObject("microformat")
                     .getObject("microformatDataRenderer")
                     .getObject("thumbnail")
-                    .getArray("thumbnails")
+                    .getArray("thumbnails");
-                    .getObject(0)
-                    .getString("url");
 
-            if (isNullOrEmpty(url)) {
+        if (!isNullOrEmpty(microFormatThumbnailsArray)) {
-                throw new ParsingException("Could not get playlist thumbnail");
+            return getImagesFromThumbnailsArray(microFormatThumbnailsArray);
-            }
         }
 
-        return fixThumbnailUrl(url);
+        throw new ParsingException("Could not get playlist thumbnails");
     }
 
     @Override
@@ -220,23 +218,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
     }
 
+    @Nonnull
     @Override
-    public String getUploaderAvatarUrl() throws ParsingException {
+    public List<Image> getUploaderAvatars() throws ParsingException {
         if (isNewPlaylistInterface) {
             // The new playlist interface doesn't provide an uploader avatar
-            return "";
+            return List.of();
         }
 
         try {
-            final String url = getUploaderInfo()
+            return getImagesFromThumbnailsArray(getUploaderInfo().getObject("thumbnail")
-                    .getObject("thumbnail")
+                    .getArray("thumbnails"));
-                    .getArray("thumbnails")
-                    .getObject(0)
-                    .getString("url");
-
-            return fixThumbnailUrl(url);
         } catch (final Exception e) {
-            throw new ParsingException("Could not get playlist uploader avatar", e);
+            throw new ParsingException("Could not get playlist uploader avatars", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -30,11 +30,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
@@ -47,6 +48,7 @@ import com.grack.nanojson.JsonWriter;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
+import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
@@ -258,23 +260,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getThumbnailUrl() throws ParsingException {
+    public List<Image> getThumbnails() throws ParsingException {
         assertPageFetched();
         try {
-            final JsonArray thumbnails = playerResponse
+            return getImagesFromThumbnailsArray(playerResponse.getObject("videoDetails")
-                    .getObject("videoDetails")
                     .getObject("thumbnail")
-                    .getArray("thumbnails");
+                    .getArray("thumbnails"));
-            // the last thumbnail is the one with the highest resolution
-            final String url = thumbnails
-                    .getObject(thumbnails.size() - 1)
-                    .getString("url");
-
-            return fixThumbnailUrl(url);
         } catch (final Exception e) {
-            throw new ParsingException("Could not get thumbnail url");
+            throw new ParsingException("Could not get thumbnails");
         }
-
     }
 
     @Nonnull
@@ -552,26 +546,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getUploaderAvatarUrl() throws ParsingException {
+    public List<Image> getUploaderAvatars() throws ParsingException {
         assertPageFetched();
 
-        final String url = getVideoSecondaryInfoRenderer()
+        final List<Image> imageList = getImagesFromThumbnailsArray(
-                .getObject("owner")
+                getVideoSecondaryInfoRenderer().getObject("owner")
-                .getObject("videoOwnerRenderer")
+                        .getObject("videoOwnerRenderer")
-                .getObject("thumbnail")
+                        .getObject("thumbnail")
-                .getArray("thumbnails")
+                        .getArray("thumbnails"));
-                .getObject(0)
-                .getString("url");
 
-        if (isNullOrEmpty(url)) {
+        if (imageList.isEmpty() && ageLimit == NO_AGE_LIMIT) {
-            if (ageLimit == NO_AGE_LIMIT) {
+            throw new ParsingException("Could not get uploader avatars");
-                throw new ParsingException("Could not get uploader avatar URL");
-            }
-
-            return "";
         }
 
-        return fixThumbnailUrl(url);
+        return imageList;
     }
 
     @Override