From d85454186a9766c73c2937a349f1b4e1d16c5f3d Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:56:25 +0200 Subject: [PATCH 01/35] Add an Image class to the extractor Objects of this serializable class contains four properties: a URL (as a string), a width, a height (represented as integers) and an estimated resolution level, which can be constructed from a given height. Possible resolution levels are: - UNKNOWN: for unknown heights or heights <= 0; - LOW: for heights > 0 & < 175; - MEDIUM: for heights >= 175 & < 720; - HIGH: for heights >= 720. Getters of these properties are available and the constructor needs these four properties. --- .../org/schabi/newpipe/extractor/Image.java | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/Image.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java new file mode 100644 index 000000000..e0d56e14a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java @@ -0,0 +1,211 @@ +package org.schabi.newpipe.extractor; + +import javax.annotation.Nonnull; +import java.io.Serializable; +import java.util.Objects; + +/** + * Class representing images in the extractor. + * + *

+ * An image has four properties: its URL, its height, its width and its estimated quality level. + *

+ * + *

+ * Depending of the services, the height, the width or both properties may be not known. + * Implementations must use the relevant unknown constants in this case + * ({@link #HEIGHT_UNKNOWN} and {@link #WIDTH_UNKNOWN}), to ensure properly the lack of knowledge + * of one or both of these properties to extractor clients. + *

+ * + *

+ * They should also respect the ranges defined in the estimated image resolution levels as much as + * possible, to ensure consistency to extractor clients. + *

+ */ +public final class Image implements Serializable { + + /** + * Constant representing that the height of an {@link Image} is unknown. + */ + public static final int HEIGHT_UNKNOWN = -1; + + /** + * Constant representing that the width of an {@link Image} is unknown. + */ + public static final int WIDTH_UNKNOWN = -1; + + @Nonnull + private final String url; + private final int height; + private final int width; + @Nonnull + private final ResolutionLevel estimatedResolutionLevel; + + /** + * Construct an {@link Image} instance. + * + * @param url the URL to the image, which should be not null or empty + * @param height the image's height + * @param width the image's width + * @param estimatedResolutionLevel the image's estimated resolution level, which must not be + * null + * @throws NullPointerException if {@code estimatedResolutionLevel} is null + */ + public Image(@Nonnull final String url, + final int height, + final int width, + @Nonnull final ResolutionLevel estimatedResolutionLevel) + throws NullPointerException { + this.url = url; + this.height = height; + this.width = width; + this.estimatedResolutionLevel = Objects.requireNonNull( + estimatedResolutionLevel, "estimatedResolutionLevel is null"); + } + + /** + * Get the URL of this {@link Image}. + * + * @return the {@link Image}'s URL. + */ + @Nonnull + public String getUrl() { + return url; + } + + /** + * Get the height of this {@link Image}. + * + *

+ * If it is unknown, {@link #HEIGHT_UNKNOWN} is returned instead. + *

+ * + * @return the {@link Image}'s height or {@link #HEIGHT_UNKNOWN} + */ + public int getHeight() { + return height; + } + + /** + * Get the width of this {@link Image}. + * + *

+ * If it is unknown, {@link #WIDTH_UNKNOWN} is returned instead. + *

+ * + * @return the {@link Image}'s width or {@link #WIDTH_UNKNOWN} + */ + public int getWidth() { + return width; + } + + /** + * Get the estimated resolution level of this image. + * + *

+ * If it is unknown, {@link ResolutionLevel#UNKNOWN} is returned instead. + *

+ * + * @return the estimated resolution level, which is never {@code null} + * @see ResolutionLevel + */ + @Nonnull + public ResolutionLevel getEstimatedResolutionLevel() { + return estimatedResolutionLevel; + } + + /** + * Get a string representation of this {@link Image} instance. + * + *

+ * The representation will be in the following format, where {@code url}, {@code height}, + * {@code width} and {@code estimatedResolutionLevel} represent the corresponding properties: + *
+ *
+ * {@code Image {url=url, height='height, width=width, + * estimatedResolutionLevel=estimatedResolutionLevel}'} + *

+ * + * @return a string representation of this {@link Image} instance + */ + @Nonnull + @Override + public String toString() { + return "Image {" + "url=" + url + ", height=" + height + ", width=" + width + + ", estimatedResolutionLevel=" + estimatedResolutionLevel + "}"; + } + + /** + * The estimated resolution level of an {@link Image}. + * + *

+ * Some services don't return the size of their images, but we may know for a specific image + * type that a service returns, according to real data, an approximation of the resolution + * level. + *

+ */ + public enum ResolutionLevel { + + /** + * The high resolution level. + * + *

+ * This level applies to images with a height greater than or equal to 720px. + *

+ */ + HIGH, + + /** + * The medium resolution level. + * + *

+ * This level applies to images with a height between 175px inclusive and 720px exclusive. + *

+ */ + MEDIUM, + + /** + * The low resolution level. + * + *

+ * This level applies to images with a height between 1px inclusive and 175px exclusive. + *

+ */ + LOW, + + /** + * The unknown resolution level. + * + *

+ * This value is returned when the extractor doesn't know what resolution level an image + * could have, for example if the extractor loops in an array of images with different + * resolution levels without knowing the height. + *

+ */ + UNKNOWN; + + /** + * Get a {@link ResolutionLevel} based from the given height. + * + * @param heightPx the height from which returning the good {@link ResolutionLevel} + * @return the {@link ResolutionLevel} corresponding to the height provided. See the + * {@link ResolutionLevel} values for details about what value is returned. + */ + public static ResolutionLevel fromHeight(final int heightPx) { + if (heightPx <= 0) { + return UNKNOWN; + } + + if (heightPx < 175) { + return LOW; + } + + if (heightPx < 720) { + return MEDIUM; + } + + return HIGH; + } + } +} From 78ce65769fe39bcdba7f9532b8f240ca2e8b4837 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:56:25 +0200 Subject: [PATCH 02/35] Add an ImageSuffix class to the extractor The goal of this utility class is to simply store suffixes which need to be appended to image URLs, in order to get images at the suffix resolution. This class contains four properties: the suffix (as a string), the height, the width (as integers) and the estimated resolution level of the image corresponding to the one represented by the suffix. --- .../newpipe/extractor/utils/ImageSuffix.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java new file mode 100644 index 000000000..d1ba7359c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java @@ -0,0 +1,104 @@ +package org.schabi.newpipe.extractor.utils; + +import org.schabi.newpipe.extractor.Image.ResolutionLevel; + +import javax.annotation.Nonnull; +import java.io.Serializable; +import java.util.Objects; + +/** + * Serializable class representing a suffix (including its format extension, such as {@code .jpg}) + * which needs to be added to get an image/thumbnail URL with its corresponding height, width and + * estimated resolution level. + * + *

+ * This class is used to construct {@link org.schabi.newpipe.extractor.Image Image} + * instances from a single base URL/path, in order to get all or most image resolutions provided, + * depending of the service and the resolutions provided. + *

+ * + *

+ * Note that this class is not intended to be used externally and so should only be used when + * interfacing with the extractor. + *

+ */ +public final class ImageSuffix implements Serializable { + @Nonnull + private final String suffix; + private final int height; + private final int width; + @Nonnull + private final ResolutionLevel resolutionLevel; + + /** + * Create a new {@link ImageSuffix} instance. + * + * @param suffix the suffix string + * @param height the height corresponding to the image suffix + * @param width the width corresponding to the image suffix + * @param estimatedResolutionLevel the {@link ResolutionLevel} of the image suffix, which must + * not be null + * @throws NullPointerException if {@code estimatedResolutionLevel} is {@code null} + */ + public ImageSuffix(@Nonnull final String suffix, + final int height, + final int width, + @Nonnull final ResolutionLevel estimatedResolutionLevel) + throws NullPointerException { + this.suffix = suffix; + this.height = height; + this.width = width; + this.resolutionLevel = Objects.requireNonNull(estimatedResolutionLevel, + "estimatedResolutionLevel is null"); + } + + /** + * @return the suffix which needs to be appended to get the full image URL + */ + @Nonnull + public String getSuffix() { + return suffix; + } + + /** + * @return the height corresponding to the image suffix, which may be unknown + */ + public int getHeight() { + return height; + } + + /** + * @return the width corresponding to the image suffix, which may be unknown + */ + public int getWidth() { + return width; + } + + /** + * @return the estimated {@link ResolutionLevel} of the suffix, which is never null. + */ + @Nonnull + public ResolutionLevel getResolutionLevel() { + return resolutionLevel; + } + + /** + * Get a string representation of this {@link ImageSuffix} instance. + * + *

+ * The representation will be in the following format, where {@code suffix}, {@code height}, + * {@code width} and {@code resolutionLevel} represent the corresponding properties: + *
+ *
+ * {@code ImageSuffix {url=url, height=height, width=width, resolutionLevel=resolutionLevel}'} + *

+ * + * @return a string representation of this {@link ImageSuffix} instance + */ + @Nonnull + @Override + public String toString() { + return "ImageSuffix {" + "suffix=" + suffix + ", height=" + height + ", width=" + + width + ", resolutionLevel=" + resolutionLevel + "}"; + } +} From 2f3ee8a3f2adf1cfd0f99542054c5becb8fa9d98 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:28:06 +0200 Subject: [PATCH 03/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItems --- .../schabi/newpipe/extractor/InfoItem.java | 26 ++++++++------- .../extractor/comments/CommentsInfoItem.java | 19 +++++++---- .../extractor/stream/StreamInfoItem.java | 32 +++++++++++-------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index cbcab0a50..3925911d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -1,33 +1,36 @@ -package org.schabi.newpipe.extractor; - /* * Created by Christian Schabesberger on 11.02.17. * * Copyright (C) Christian Schabesberger 2017 - * InfoItem.java is part of NewPipe. + * InfoItem.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor; + +import javax.annotation.Nonnull; import java.io.Serializable; +import java.util.List; public abstract class InfoItem implements Serializable { private final InfoType infoType; private final int serviceId; private final String url; private final String name; - private String thumbnailUrl; + @Nonnull + private List thumbnails = List.of(); public InfoItem(final InfoType infoType, final int serviceId, @@ -55,12 +58,13 @@ public abstract class InfoItem implements Serializable { return name; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java index 0752e9b74..1b679f2cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java @@ -1,18 +1,22 @@ package org.schabi.newpipe.extractor.comments; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; public class CommentsInfoItem extends InfoItem { private String commentId; private Description commentText; private String uploaderName; - private String uploaderAvatarUrl; + @Nonnull + private List uploaderAvatars = List.of(); private String uploaderUrl; private boolean uploaderVerified; private String textualUploadDate; @@ -60,12 +64,13 @@ public class CommentsInfoItem extends InfoItem { this.uploaderName = uploaderName; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getUploaderUrl() { @@ -94,8 +99,8 @@ public class CommentsInfoItem extends InfoItem { } /** - * @return the comment's like count - * or {@link CommentsInfoItem#NO_LIKE_COUNT} if it is unavailable + * @return the comment's like count or {@link CommentsInfoItem#NO_LIKE_COUNT} if it is + * unavailable */ public int getLikeCount() { return likeCount; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java index 37adb475f..a478a6994 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java @@ -1,32 +1,35 @@ -package org.schabi.newpipe.extractor.stream; - /* * Created by Christian Schabesberger on 26.08.15. * * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItem.java is part of NewPipe. + * StreamInfoItem.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.localization.DateWrapper; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; /** - * Info object for previews of unopened videos, eg search results, related videos + * Info object for previews of unopened videos, e.g. search results, related videos. */ public class StreamInfoItem extends InfoItem { private final StreamType streamType; @@ -40,7 +43,8 @@ public class StreamInfoItem extends InfoItem { private long duration = -1; private String uploaderUrl = null; - private String uploaderAvatarUrl = null; + @Nonnull + private List uploaderAvatars = List.of(); private boolean uploaderVerified = false; private boolean shortFormContent = false; @@ -88,13 +92,13 @@ public class StreamInfoItem extends InfoItem { this.uploaderUrl = uploaderUrl; } - @Nullable - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getShortDescription() { @@ -152,7 +156,7 @@ public class StreamInfoItem extends InfoItem { + ", serviceId=" + getServiceId() + ", url='" + getUrl() + '\'' + ", name='" + getName() + '\'' - + ", thumbnailUrl='" + getThumbnailUrl() + '\'' + + ", thumbnails='" + getThumbnails() + '\'' + ", uploaderVerified='" + isUploaderVerified() + '\'' + '}'; } From ca1d4a6fa4e3c679dba514098c409a463d676af8 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:31:47 +0200 Subject: [PATCH 04/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItemExtractors --- .../newpipe/extractor/InfoItemExtractor.java | 6 +- .../comments/CommentsInfoItemExtractor.java | 8 ++- .../stream/StreamInfoItemExtractor.java | 55 ++++++++++--------- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java index 0e38ceecd..ed5974176 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java @@ -2,8 +2,12 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + public interface InfoItemExtractor { String getName() throws ParsingException; String getUrl() throws ParsingException; - String getThumbnailUrl() throws ParsingException; + @Nonnull + List getThumbnails() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 695478764..dfff3bda4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.comments; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -8,7 +9,9 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsI import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; public interface CommentsInfoItemExtractor extends InfoItemExtractor { @@ -77,8 +80,9 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor { return ""; } - default String getUploaderAvatarUrl() throws ParsingException { - return ""; + @Nonnull + default List getUploaderAvatars() throws ParsingException { + return List.of(); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 77a633e2d..62e69a433 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -1,39 +1,41 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.InfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.DateWrapper; - -import javax.annotation.Nullable; - /* * Created by Christian Schabesberger on 28.02.16. * * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemExtractor.java is part of NewPipe. + * StreamInfoItemExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ -public interface StreamInfoItemExtractor extends InfoItemExtractor { +package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream type * * @return the stream type - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ StreamType getStreamType() throws ParsingException; @@ -41,7 +43,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Check if the stream is an ad. * * @return {@code true} if the stream is an ad. - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ boolean isAd() throws ParsingException; @@ -49,7 +51,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Get the stream duration in seconds * * @return the stream duration in seconds - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ long getDuration() throws ParsingException; @@ -57,7 +59,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Parses the number of views * * @return the number of views or -1 for live streams - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ long getViewCount() throws ParsingException; @@ -65,27 +67,29 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Get the uploader name * * @return the uploader name - * @throws ParsingException if parsing fails + * @throws ParsingException if there is an error in the extraction */ String getUploaderName() throws ParsingException; String getUploaderUrl() throws ParsingException; /** - * Get the uploader's avatar + * Get the uploader avatars. * - * @return The uploader's avatar url or {@code null} if not provided by the service. + * @return the uploader avatars or an empty list if not provided by the service * @throws ParsingException if there is an error in the extraction */ - @Nullable - String getUploaderAvatarUrl() throws ParsingException; + @Nonnull + default List getUploaderAvatars() throws ParsingException { + return List.of(); + } /** * Whether the uploader has been verified by the service's provider. * If there is no verification implemented, return false. * * @return whether the uploader has been verified by the service's provider - * @throws ParsingException + * @throws ParsingException if there is an error in the extraction */ boolean isUploaderVerified() throws ParsingException; @@ -109,8 +113,8 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { *

* * @return The date and time (can be approximated) this item was uploaded or {@code null}. - * @throws ParsingException if there is an error in the extraction - * or the extracted date couldn't be parsed. + * @throws ParsingException if there is an error in the extraction or the extracted date + * couldn't be parsed * @see #getTextualUploadDate() */ @Nullable @@ -137,6 +141,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { *

* * @return whether the stream is a short-form content + * @throws ParsingException if there is an error in the extraction */ default boolean isShortFormContent() throws ParsingException { return false; From 0f4a5a81841df6c03352a4c9778b36d808d779c0 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:36:27 +0200 Subject: [PATCH 05/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItemsCollectors --- .../channel/ChannelInfoItemsCollector.java | 20 ++++---- .../comments/CommentsInfoItemsCollector.java | 4 +- .../playlist/PlaylistInfoItemsCollector.java | 2 +- .../stream/StreamInfoItemsCollector.java | 47 +++++++++---------- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java index 5b085f8b8..5b8ff6918 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java @@ -1,28 +1,28 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - /* * Created by Christian Schabesberger on 12.02.17. * * Copyright (C) Christian Schabesberger 2017 - * ChannelInfoItemsCollector.java is part of NewPipe. + * ChannelInfoItemsCollector.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.channel; + +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + public final class ChannelInfoItemsCollector extends InfoItemsCollector { public ChannelInfoItemsCollector(final int serviceId) { @@ -47,7 +47,7 @@ public final class ChannelInfoItemsCollector addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java index 3afeb0455..fca5bdf59 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java @@ -36,7 +36,7 @@ public final class CommentsInfoItemsCollector addError(e); } try { - resultItem.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + resultItem.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { addError(e); } @@ -66,7 +66,7 @@ public final class CommentsInfoItemsCollector addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java index 4fe35e40e..0b854a999 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java @@ -32,7 +32,7 @@ public class PlaylistInfoItemsCollector addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 35cf7fd32..c0e1ac1e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -1,3 +1,23 @@ +/* + * Created by Christian Schabesberger on 28.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * StreamInfoItemsCollector.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.InfoItemsCollector; @@ -6,26 +26,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.util.Comparator; -/* - * Created by Christian Schabesberger on 28.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemsCollector.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class StreamInfoItemsCollector extends InfoItemsCollector { @@ -74,7 +74,7 @@ public class StreamInfoItemsCollector addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } @@ -84,7 +84,7 @@ public class StreamInfoItemsCollector addError(e); } try { - resultItem.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + resultItem.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { addError(e); } @@ -111,8 +111,7 @@ public class StreamInfoItemsCollector public void commit(final StreamInfoItemExtractor extractor) { try { addItem(extract(extractor)); - } catch (final FoundAdException ae) { - //System.out.println("AD_WARNING: " + ae.getMessage()); + } catch (final FoundAdException ignored) { } catch (final Exception e) { addError(e); } From 9d8098576ea3da32277cf417ff4e52acfeeb176a Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 21 Jul 2022 00:17:45 +0200 Subject: [PATCH 06/35] Replace avatar and thumbnail URLs attributes and methods to List in Extractors --- .../extractor/channel/ChannelExtractor.java | 50 +++++++++-------- .../extractor/playlist/PlaylistExtractor.java | 21 ++++---- .../extractor/stream/StreamExtractor.java | 53 ++++++++++++------- 3 files changed, 72 insertions(+), 52 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index db0333647..d5587ec4e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,6 +1,27 @@ +/* + * Created by Christian Schabesberger on 25.07.16. + * + * Copyright (C) Christian Schabesberger 2016 + * ChannelExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -8,26 +29,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import javax.annotation.Nonnull; import java.util.List; -/* - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ChannelExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public abstract class ChannelExtractor extends Extractor { public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; @@ -36,14 +37,17 @@ public abstract class ChannelExtractor extends Extractor { super(service, linkHandler); } - public abstract String getAvatarUrl() throws ParsingException; - public abstract String getBannerUrl() throws ParsingException; + @Nonnull + public abstract List getAvatars() throws ParsingException; + @Nonnull + public abstract List getBanners() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; public abstract String getParentChannelName() throws ParsingException; public abstract String getParentChannelUrl() throws ParsingException; - public abstract String getParentChannelAvatarUrl() throws ParsingException; + @Nonnull + public abstract List getParentChannelAvatars() throws ParsingException; public abstract boolean isVerified() throws ParsingException; @Nonnull public abstract List getTabs() throws ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index bc4eee467..a714deadb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.playlist; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -9,6 +10,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.List; + public abstract class PlaylistExtractor extends ListExtractor { public PlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { @@ -17,7 +21,8 @@ public abstract class PlaylistExtractor extends ListExtractor { public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderName() throws ParsingException; - public abstract String getUploaderAvatarUrl() throws ParsingException; + @Nonnull + public abstract List getUploaderAvatars() throws ParsingException; public abstract boolean isUploaderVerified() throws ParsingException; public abstract long getStreamCount() throws ParsingException; @@ -26,15 +31,13 @@ public abstract class PlaylistExtractor extends ListExtractor { public abstract Description getDescription() throws ParsingException; @Nonnull - public String getThumbnailUrl() throws ParsingException { - return ""; + public List getThumbnails() throws ParsingException { + return Collections.emptyList(); } @Nonnull - public String getBannerUrl() throws ParsingException { - // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement it in the frontend. - return ""; + public List getBanners() throws ParsingException { + return List.of(); } @Nonnull @@ -48,8 +51,8 @@ public abstract class PlaylistExtractor extends ListExtractor { } @Nonnull - public String getSubChannelAvatarUrl() throws ParsingException { - return ""; + public List getSubChannelAvatars() throws ParsingException { + return List.of(); } public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 6fbcf8fbb..f974cade0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -1,25 +1,26 @@ -package org.schabi.newpipe.extractor.stream; - /* * Created by Christian Schabesberger on 10.08.18. * * Copyright (C) Christian Schabesberger 2016 - * StreamExtractor.java is part of NewPipe. + * StreamExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.InfoItemExtractor; @@ -87,13 +88,12 @@ public abstract class StreamExtractor extends Extractor { } /** - * This will return the url to the thumbnail of the stream. Try to return the medium resolution - * here. + * This will return the thumbnails of the stream. * - * @return The url of the thumbnail. + * @return the thumbnails of the stream */ @Nonnull - public abstract String getThumbnailUrl() throws ParsingException; + public abstract List getThumbnails() throws ParsingException; /** * This is the stream description. @@ -208,14 +208,18 @@ public abstract class StreamExtractor extends Extractor { } /** - * The url to the image file/profile picture/avatar of the creator/uploader of the stream. - * If the url is not available you can return an empty String. + * The image files/profile pictures/avatars of the creator/uploader of the stream. * - * @return The url of the image file of the uploader or an empty String + *

+ * If they are not available in the stream on specific cases, you must return an empty list for + * these ones, like it is made by default. + *

+ * + * @return the avatars of the sub-channel of the stream or an empty list (default) */ @Nonnull - public String getUploaderAvatarUrl() throws ParsingException { - return ""; + public List getUploaderAvatars() throws ParsingException { + return List.of(); } /** @@ -243,14 +247,23 @@ public abstract class StreamExtractor extends Extractor { } /** - * The url to the image file/profile picture/avatar of the sub-channel of the stream. - * If the url is not available you can return an empty String. + * The avatars of the sub-channel of the stream. * - * @return The url of the image file of the sub-channel or an empty String + *

+ * If they are not available in the stream on specific cases, you must return an empty list for + * these ones, like it is made by default. + *

+ * + *

+ * If the concept of sub-channels doesn't apply to the stream's service, keep the default + * implementation. + *

+ * + * @return the avatars of the sub-channel of the stream or an empty list (default) */ @Nonnull - public String getSubChannelAvatarUrl() throws ParsingException { - return ""; + public List getSubChannelAvatars() throws ParsingException { + return List.of(); } /** From d56b880cae7393ed71f31deec770182352f8e412 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:22:14 +0200 Subject: [PATCH 07/35] Replace avatar and thumbnail URLs attributes and methods to List in Infos --- .../extractor/channel/ChannelInfo.java | 83 ++++++------ .../extractor/playlist/PlaylistInfo.java | 67 +++++----- .../newpipe/extractor/stream/StreamInfo.java | 123 ++++++++---------- 3 files changed, 138 insertions(+), 135 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 4c05de692..502901e29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -1,6 +1,27 @@ +/* + * Created by Christian Schabesberger on 31.07.16. + * + * Copyright (C) Christian Schabesberger 2016 + * ChannelInfo.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.Info; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -11,26 +32,6 @@ import java.util.List; import javax.annotation.Nonnull; -/* - * Created by Christian Schabesberger on 31.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ChannelInfo.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class ChannelInfo extends Info { public ChannelInfo(final int serviceId, @@ -64,13 +65,13 @@ public class ChannelInfo extends Info { final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name); try { - info.setAvatarUrl(extractor.getAvatarUrl()); + info.setAvatars(extractor.getAvatars()); } catch (final Exception e) { info.addError(e); } try { - info.setBannerUrl(extractor.getBannerUrl()); + info.setBanners(extractor.getBanners()); } catch (final Exception e) { info.addError(e); } @@ -106,7 +107,7 @@ public class ChannelInfo extends Info { } try { - info.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + info.setParentChannelAvatars(extractor.getParentChannelAvatars()); } catch (final Exception e) { info.addError(e); } @@ -132,15 +133,18 @@ public class ChannelInfo extends Info { return info; } - private String avatarUrl; private String parentChannelName; private String parentChannelUrl; - private String parentChannelAvatarUrl; - private String bannerUrl; private String feedUrl; private long subscriberCount = -1; private String description; private String[] donationLinks; + @Nonnull + private List avatars = List.of(); + @Nonnull + private List banners = List.of(); + @Nonnull + private List parentChannelAvatars = List.of(); private boolean verified; private List tabs = List.of(); private List tags = List.of(); @@ -161,28 +165,31 @@ public class ChannelInfo extends Info { this.parentChannelUrl = parentChannelUrl; } - public String getParentChannelAvatarUrl() { - return parentChannelAvatarUrl; + @Nonnull + public List getParentChannelAvatars() { + return parentChannelAvatars; } - public void setParentChannelAvatarUrl(final String parentChannelAvatarUrl) { - this.parentChannelAvatarUrl = parentChannelAvatarUrl; + public void setParentChannelAvatars(@Nonnull final List parentChannelAvatars) { + this.parentChannelAvatars = parentChannelAvatars; } - public String getAvatarUrl() { - return avatarUrl; + @Nonnull + public List getAvatars() { + return avatars; } - public void setAvatarUrl(final String avatarUrl) { - this.avatarUrl = avatarUrl; + public void setAvatars(@Nonnull final List avatars) { + this.avatars = avatars; } - public String getBannerUrl() { - return bannerUrl; + @Nonnull + public List getBanners() { + return banners; } - public void setBannerUrl(final String bannerUrl) { - this.bannerUrl = bannerUrl; + public void setBanners(@Nonnull final List banners) { + this.banners = banners; } public String getFeedUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 97a0d530d..bb29ca7d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.playlist; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; @@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -109,26 +111,23 @@ public final class PlaylistInfo extends ListInfo { info.addError(e); } try { - info.setThumbnailUrl(extractor.getThumbnailUrl()); + info.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { info.addError(e); } try { info.setUploaderUrl(extractor.getUploaderUrl()); } catch (final Exception e) { - info.setUploaderUrl(""); uploaderParsingErrors.add(e); } try { info.setUploaderName(extractor.getUploaderName()); } catch (final Exception e) { - info.setUploaderName(""); uploaderParsingErrors.add(e); } try { - info.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + info.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { - info.setUploaderAvatarUrl(""); uploaderParsingErrors.add(e); } try { @@ -142,12 +141,12 @@ public final class PlaylistInfo extends ListInfo { uploaderParsingErrors.add(e); } try { - info.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + info.setSubChannelAvatars(extractor.getSubChannelAvatars()); } catch (final Exception e) { uploaderParsingErrors.add(e); } try { - info.setBannerUrl(extractor.getBannerUrl()); + info.setBanners(extractor.getBanners()); } catch (final Exception e) { info.addError(e); } @@ -171,32 +170,38 @@ public final class PlaylistInfo extends ListInfo { return info; } - private String thumbnailUrl; - private String bannerUrl; - private String uploaderUrl; - private String uploaderName; - private String uploaderAvatarUrl; + private String uploaderUrl = ""; + private String uploaderName = ""; private String subChannelUrl; private String subChannelName; - private String subChannelAvatarUrl; - private long streamCount = 0; private Description description; + @Nonnull + private List banners = List.of(); + @Nonnull + private List subChannelAvatars = List.of(); + @Nonnull + private List thumbnails = List.of(); + @Nonnull + private List uploaderAvatars = List.of(); + private long streamCount; private PlaylistType playlistType; - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } - public String getBannerUrl() { - return bannerUrl; + @Nonnull + public List getBanners() { + return banners; } - public void setBannerUrl(final String bannerUrl) { - this.bannerUrl = bannerUrl; + public void setBanners(@Nonnull final List banners) { + this.banners = banners; } public String getUploaderUrl() { @@ -215,12 +220,13 @@ public final class PlaylistInfo extends ListInfo { this.uploaderName = uploaderName; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getSubChannelUrl() { @@ -239,12 +245,13 @@ public final class PlaylistInfo extends ListInfo { this.subChannelName = subChannelName; } - public String getSubChannelAvatarUrl() { - return subChannelAvatarUrl; + @Nonnull + public List getSubChannelAvatars() { + return subChannelAvatars; } - public void setSubChannelAvatarUrl(final String subChannelAvatarUrl) { - this.subChannelAvatarUrl = subChannelAvatarUrl; + public void setSubChannelAvatars(@Nonnull final List subChannelAvatars) { + this.subChannelAvatars = subChannelAvatars; } public long getStreamCount() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index fa979c34b..252bc7f14 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -1,26 +1,3 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.Info; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.MetaInfo; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.ExtractorHelper; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - /* * Created by Christian Schabesberger on 26.08.15. * @@ -38,9 +15,31 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Info; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + /** * Info object for opened contents, i.e. the content ready to play. */ @@ -106,9 +105,7 @@ public class StreamInfo extends Info { // Important data, without it the content can't be displayed. // If one of these is not available, the frontend will receive an exception directly. - final int serviceId = extractor.getServiceId(); final String url = extractor.getUrl(); - final String originalUrl = extractor.getOriginalUrl(); final StreamType streamType = extractor.getStreamType(); final String id = extractor.getId(); final String name = extractor.getName(); @@ -148,7 +145,6 @@ public class StreamInfo extends Info { streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e)); } - /* Load and extract audio */ try { streamInfo.setAudioStreams(extractor.getAudioStreams()); } catch (final ContentNotSupportedException e) { @@ -157,31 +153,18 @@ public class StreamInfo extends Info { streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); } - /* Extract video stream url */ try { streamInfo.setVideoStreams(extractor.getVideoStreams()); } catch (final Exception e) { streamInfo.addError(new ExtractionException("Couldn't get video streams", e)); } - /* Extract video only stream url */ try { streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams()); } catch (final Exception e) { streamInfo.addError(new ExtractionException("Couldn't get video only streams", e)); } - // Lists can be null if an exception was thrown during extraction - if (streamInfo.getVideoStreams() == null) { - streamInfo.setVideoStreams(Collections.emptyList()); - } - if (streamInfo.getVideoOnlyStreams() == null) { - streamInfo.setVideoOnlyStreams(Collections.emptyList()); - } - if (streamInfo.getAudioStreams() == null) { - streamInfo.setAudioStreams(Collections.emptyList()); - } - // Either audio or video has to be available, otherwise we didn't get a stream (since // videoOnly are optional, they don't count). if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) { @@ -199,7 +182,7 @@ public class StreamInfo extends Info { // so the frontend can afterwards check where errors happened. try { - streamInfo.setThumbnailUrl(extractor.getThumbnailUrl()); + streamInfo.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { streamInfo.addError(e); } @@ -219,7 +202,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + streamInfo.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { streamInfo.addError(e); } @@ -245,7 +228,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + streamInfo.setSubChannelAvatars(extractor.getSubChannelAvatars()); } catch (final Exception e) { streamInfo.addError(e); } @@ -353,7 +336,8 @@ public class StreamInfo extends Info { } private StreamType streamType; - private String thumbnailUrl = ""; + @Nonnull + private List thumbnails = List.of(); private String textualUploadDate; private DateWrapper uploadDate; private long duration = -1; @@ -366,24 +350,26 @@ public class StreamInfo extends Info { private String uploaderName = ""; private String uploaderUrl = ""; - private String uploaderAvatarUrl = ""; + @Nonnull + private List uploaderAvatars = List.of(); private boolean uploaderVerified = false; private long uploaderSubscriberCount = -1; private String subChannelName = ""; private String subChannelUrl = ""; - private String subChannelAvatarUrl = ""; + @Nonnull + private List subChannelAvatars = List.of(); - private List videoStreams = new ArrayList<>(); - private List audioStreams = new ArrayList<>(); - private List videoOnlyStreams = new ArrayList<>(); + private List videoStreams = List.of(); + private List audioStreams = List.of(); + private List videoOnlyStreams = List.of(); private String dashMpdUrl = ""; private String hlsUrl = ""; - private List relatedItems = new ArrayList<>(); + private List relatedItems = List.of(); private long startPosition = 0; - private List subtitles = new ArrayList<>(); + private List subtitles = List.of(); private String host = ""; private StreamExtractor.Privacy privacy; @@ -391,15 +377,15 @@ public class StreamInfo extends Info { private String licence = ""; private String supportInfo = ""; private Locale language = null; - private List tags = new ArrayList<>(); - private List streamSegments = new ArrayList<>(); - private List metaInfo = new ArrayList<>(); + private List tags = List.of(); + private List streamSegments = List.of(); + private List metaInfo = List.of(); private boolean shortFormContent = false; /** * Preview frames, e.g. for the storyboard / seekbar thumbnail preview */ - private List previewFrames = Collections.emptyList(); + private List previewFrames = List.of(); /** * Get the stream type @@ -419,12 +405,13 @@ public class StreamInfo extends Info { * * @return the thumbnail url as a string */ - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } public String getTextualUploadDate() { @@ -522,12 +509,13 @@ public class StreamInfo extends Info { this.uploaderUrl = uploaderUrl; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public boolean isUploaderVerified() { @@ -562,12 +550,13 @@ public class StreamInfo extends Info { this.subChannelUrl = subChannelUrl; } - public String getSubChannelAvatarUrl() { - return subChannelAvatarUrl; + @Nonnull + public List getSubChannelAvatars() { + return subChannelAvatars; } - public void setSubChannelAvatarUrl(final String subChannelAvatarUrl) { - this.subChannelAvatarUrl = subChannelAvatarUrl; + public void setSubChannelAvatars(@Nonnull final List subChannelAvatars) { + this.subChannelAvatars = subChannelAvatars; } public List getVideoStreams() { From adfad086ac21a8aa7af375bbf16ab2761592dc64 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 17:28:39 +0200 Subject: [PATCH 08/35] [YouTube] Add utility methods to get images from InfoItems and thumbnails arrays Unmodifiable lists of Images are returned, parsed from a given YouTube "thumbnails" JSON array. These methods will be used in all YouTube extractors and InfoItems, as the structures between content types (videos, channels, playlists, ...) are common. --- .../youtube/YoutubeParsingHelper.java | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index aec550897..1bce2c2f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -33,6 +33,9 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.jsoup.nodes.Entities; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException; @@ -69,6 +72,7 @@ import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -1133,17 +1137,61 @@ public final class YoutubeParsingHelper { return result; } - public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem) + /** + * Get thumbnails from a {@link JsonObject} representing a YouTube + * {@link org.schabi.newpipe.extractor.InfoItem InfoItem}. + * + *

+ * Thumbnails are got from the {@code thumbnails} {@link JsonArray} inside the {@code thumbnail} + * {@link JsonObject} of the YouTube {@link org.schabi.newpipe.extractor.InfoItem InfoItem}, + * using {@link #getImagesFromThumbnailsArray(JsonArray)}. + *

+ * + * @param infoItem a YouTube {@link org.schabi.newpipe.extractor.InfoItem InfoItem} + * @return an unmodifiable list of {@link Image}s found in the {@code thumbnails} + * {@link JsonArray} + * @throws ParsingException if an exception occurs when + * {@link #getImagesFromThumbnailsArray(JsonArray)} is executed + */ + @Nonnull + public static List getThumbnailsFromInfoItem(@Nonnull final JsonObject infoItem) throws ParsingException { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution try { - return fixThumbnailUrl(infoItem.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url")); + return getImagesFromThumbnailsArray(infoItem.getObject("thumbnail") + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails from InfoItem", e); } } + /** + * Get images from a YouTube {@code thumbnails} {@link JsonArray}. + * + *

+ * The properties of the {@link Image}s created will be set using the corresponding ones of + * thumbnail items. + *

+ * + * @param thumbnails a YouTube {@code thumbnails} {@link JsonArray} + * @return an unmodifiable list of {@link Image}s extracted from the given {@link JsonArray} + */ + @Nonnull + public static List getImagesFromThumbnailsArray( + @Nonnull final JsonArray thumbnails) { + return thumbnails.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(thumbnail -> !isNullOrEmpty(thumbnail.getString("url"))) + .map(thumbnail -> { + final int height = thumbnail.getInt("height", Image.HEIGHT_UNKNOWN); + return new Image(fixThumbnailUrl(thumbnail.getString("url")), + height, + thumbnail.getInt("width", Image.WIDTH_UNKNOWN), + ResolutionLevel.fromHeight(height)); + }) + .collect(Collectors.toUnmodifiableList()); + } + @Nonnull public static String getValidJsonResponseBody(@Nonnull final Response response) throws ParsingException, MalformedURLException { From 4cc99f9ce1ba059441222b95386af722f6b19658 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 21:34:12 +0200 Subject: [PATCH 09/35] [YouTube] Apply changes in InfoItemExtractors except YouTube Music ones --- .../YoutubeChannelInfoItemExtractor.java | 56 +++++++------- .../YoutubeCommentsInfoItemExtractor.java | 41 +++++----- .../YoutubeFeedInfoItemExtractor.java | 55 ++++++++++--- ...YoutubeMixOrPlaylistInfoItemExtractor.java | 11 ++- .../YoutubePlaylistInfoItemExtractor.java | 15 ++-- .../YoutubeReelInfoItemExtractor.java | 19 +++-- .../YoutubeStreamInfoItemExtractor.java | 77 ++++++++++--------- 7 files changed, 166 insertions(+), 108 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index e01f0cfb9..302fb3ef9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,7 +1,28 @@ +/* + * Created by Christian Schabesberger on 12.02.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeChannelInfoItemExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -9,28 +30,11 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import javax.annotation.Nonnull; +import java.util.List; -/* - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeChannelInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject channelInfoItem; @@ -53,15 +57,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor this.withHandle = wHandle; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { - final String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - - return fixThumbnailUrl(url); + return getThumbnailsFromInfoItem(channelInfoItem); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 95209a65f..fb6463a54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -11,9 +12,12 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import static org.schabi.newpipe.extractor.comments.CommentsInfoItem.UNKNOWN_REPLY_COUNT; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -42,20 +46,25 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract return commentRenderer; } + @Nonnull + private List getAuthorThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray(JsonUtils.getArray(getCommentRenderer(), + "authorThumbnail.thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get author thumbnails", e); + } + } + @Override public String getUrl() throws ParsingException { return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), - "authorThumbnail.thumbnails"); - return JsonUtils.getString(arr.getObject(2), "url"); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } + public List getThumbnails() throws ParsingException { + return getAuthorThumbnails(); } @Override @@ -204,15 +213,10 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { - try { - final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), - "authorThumbnail.thumbnails"); - return JsonUtils.getString(arr.getObject(2), "url"); - } catch (final Exception e) { - throw new ParsingException("Could not get author thumbnail", e); - } + public List getUploaderAvatars() throws ParsingException { + return getAuthorThumbnails(); } @Override @@ -228,6 +232,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract return getCommentRenderer().has("pinnedCommentBadge"); } + @Override public boolean isUploaderVerified() throws ParsingException { return getCommentRenderer().has("authorCommentBadge"); } @@ -261,7 +266,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract } @Override - public Page getReplies() throws ParsingException { + public Page getReplies() { try { final String id = JsonUtils.getString( JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index d5a83d3b7..d917eb2d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -1,14 +1,18 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import java.util.List; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { private final Element entryElement; @@ -51,12 +55,6 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { return entryElement.select("author > uri").first().text(); } - @Nullable - @Override - public String getUploaderAvatarUrl() throws ParsingException { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; @@ -89,12 +87,51 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { return entryElement.getElementsByTag("link").first().attr("href"); } + @Nonnull @Override - public String getThumbnailUrl() { + public List getThumbnails() { + final Element thumbnailElement = entryElement.getElementsByTag("media:thumbnail").first(); + if (thumbnailElement == null) { + return List.of(); + } + + final String feedThumbnailUrl = thumbnailElement.attr("url"); + + // If the thumbnail URL is empty, it means that no thumbnail is available, return an empty + // list in this case + if (feedThumbnailUrl.isEmpty()) { + return List.of(); + } + // The hqdefault thumbnail has some black bars at the top and at the bottom, while the // mqdefault doesn't, so return the mqdefault one. It should always exist, according to // https://stackoverflow.com/a/20542029/9481500. - return entryElement.getElementsByTag("media:thumbnail").first().attr("url") - .replace("hqdefault", "mqdefault"); + final String newFeedThumbnailUrl = feedThumbnailUrl.replace("hqdefault", "mqdefault"); + + int height; + int width; + + // If the new thumbnail URL is equal to the feed one, it means that a different image + // resolution is used on feeds, so use the height and width provided instead of the + // mqdefault ones + if (newFeedThumbnailUrl.equals(feedThumbnailUrl)) { + try { + height = Integer.parseInt(thumbnailElement.attr("height")); + } catch (final NumberFormatException e) { + height = Image.HEIGHT_UNKNOWN; + } + + try { + width = Integer.parseInt(thumbnailElement.attr("width")); + } catch (final NumberFormatException e) { + width = Image.WIDTH_UNKNOWN; + } + } else { + height = 320; + width = 180; + } + + return List.of( + new Image(newFeedThumbnailUrl, height, width, ResolutionLevel.fromHeight(height))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 4847302e6..16c7a3e3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -2,11 +2,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; @@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import javax.annotation.Nonnull; +import java.util.List; public class YoutubeMixOrPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject mixInfoItem; @@ -40,9 +42,10 @@ public class YoutubeMixOrPlaylistInfoItemExtractor implements PlaylistInfoItemEx return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(mixInfoItem); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(mixInfoItem); } @Override @@ -75,7 +78,7 @@ public class YoutubeMixOrPlaylistInfoItemExtractor implements PlaylistInfoItemEx return Integer.parseInt(countString); } catch (final NumberFormatException ignored) { // un-parsable integer: this is a mix with infinite items and "50+" as count string - // (though youtube music mixes do not necessarily have an infinite count of songs) + // (though YouTube Music mixes do not necessarily have an infinite count of songs) return ListExtractor.ITEM_COUNT_INFINITE; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index c3a7707dc..a0584a20f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -2,17 +2,21 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; - public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject playlistInfoItem; @@ -20,8 +24,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract this.playlistInfoItem = playlistInfoItem; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { JsonArray thumbnails = playlistInfoItem.getArray("thumbnails") .getObject(0) @@ -31,9 +36,9 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract .getArray("thumbnails"); } - return fixThumbnailUrl(thumbnails.getObject(0).getString("url")); + return getImagesFromThumbnailsArray(thumbnails); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index 1b68c2ddf..911cb4bed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -13,9 +15,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import java.util.List; + /** * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}. * @@ -53,9 +57,10 @@ public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor { } } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(reelInfo); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(reelInfo); } @Override @@ -101,7 +106,7 @@ public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor { } @Override - public boolean isShortFormContent() throws ParsingException { + public boolean isShortFormContent() { return true; } @@ -122,12 +127,6 @@ public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor { return null; } - @Nullable - @Override - public String getUploaderAvatarUrl() throws ParsingException { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 1607e0a0f..178cc2bf6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,33 @@ +/* + * Copyright (C) Christian Schabesberger 2016 + * YoutubeStreamInfoItemExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -15,35 +41,14 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; + import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.regex.Pattern; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -/* - * Copyright (C) Christian Schabesberger 2016 - * YoutubeStreamInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private static final Pattern ACCESSIBILITY_DATA_VIEW_COUNT_REGEX = @@ -215,21 +220,22 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return url; } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { if (videoInfo.has("channelThumbnailSupportedRenderers")) { - return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers" - + ".channelThumbnailWithLinkRenderer.thumbnail.thumbnails") - .getObject(0).getString("url"); + return getImagesFromThumbnailsArray(JsonUtils.getArray(videoInfo, + // CHECKSTYLE:OFF + "channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails")); + // CHECKSTYLE:ON } if (videoInfo.has("channelThumbnail")) { - return JsonUtils.getArray(videoInfo, "channelThumbnail.thumbnails") - .getObject(0).getString("url"); + return getImagesFromThumbnailsArray( + JsonUtils.getArray(videoInfo, "channelThumbnail.thumbnails")); } - return null; + return List.of(); } @Override @@ -371,9 +377,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { videoInfoTitleAccessibilityData))); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(videoInfo); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(videoInfo); } private boolean isPremium() { @@ -409,10 +416,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public String getShortDescription() throws ParsingException { - if (videoInfo.has("detailedMetadataSnippets")) { return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") - .getObject(0).getObject("snippetText")); + .getObject(0) + .getObject("snippetText")); } if (videoInfo.has("descriptionSnippet")) { From c1981ed54feba983172fc6f1e0a74cbefb0f2da4 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 22:31:43 +0200 Subject: [PATCH 10/35] [YouTube] Apply changes in Extractors except YoutubeMusicSearchExtractor Also improve a bit some code related to the changes. --- .../extractors/YoutubeChannelExtractor.java | 104 ++++++++---------- .../YoutubeMixPlaylistExtractor.java | 38 +++++-- .../extractors/YoutubePlaylistExtractor.java | 54 ++++----- .../extractors/YoutubeStreamExtractor.java | 44 +++----- 4 files changed, 110 insertions(+), 130 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 66b927e02..f35d20ebc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,3 +1,23 @@ +/* + * Created by Christian Schabesberger on 25.07.16. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeChannelExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse; @@ -8,6 +28,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -36,26 +57,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; -/* - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeChannelExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class YoutubeChannelExtractor extends ChannelExtractor { private JsonObject jsonResponse; @@ -190,16 +191,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .orElseThrow(() -> new ParsingException("Could not get channel name")); } + @Nonnull @Override - public String getAvatarUrl() throws ParsingException { + public List getAvatars() throws ParsingException { assertPageFetched(); if (channelAgeGateRenderer != null) { return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar") - .getArray("thumbnails") - .getObject(0) - .getString("url")) - .map(YoutubeParsingHelper::fixThumbnailUrl) - .orElseThrow(() -> new ParsingException("Could not get avatar URL")); + .getArray("thumbnails")) + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElseThrow(() -> new ParsingException("Could not get avatars")); } return channelHeader.map(header -> { @@ -210,56 +210,37 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getObject("image") .getObject("contentPreviewImageViewModel") .getObject("image") - .getArray("sources") - .getObject(0) - .getString("url"); + .getArray("sources"); case INTERACTIVE_TABBED: return header.json.getObject("boxArt") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); case C4_TABBED: case CAROUSEL: default: return header.json.getObject("avatar") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); } }) - .map(YoutubeParsingHelper::fixThumbnailUrl) - .orElseThrow(() -> new ParsingException("Could not get avatar URL")); + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElseThrow(() -> new ParsingException("Could not get avatars")); } + @Nonnull @Override - public String getBannerUrl() throws ParsingException { + public List getBanners() { assertPageFetched(); if (channelAgeGateRenderer != null) { - return null; + return List.of(); } - if (channelHeader.isPresent()) { - final ChannelHeader header = channelHeader.get(); - if (header.headerType == HeaderType.PAGE) { - // No banner is available on pageHeaderRenderer headers - return null; - } - - return Optional.ofNullable(header.json.getObject("banner") - .getArray("thumbnails") - .getObject(0) - .getString("url")) - .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner")) - .map(YoutubeParsingHelper::fixThumbnailUrl) - // Channels may not have a banner, so no exception should be thrown if no - // banner is found - // Return null in this case - .orElse(null); - } - - return null; + // No banner is available on pageHeaderRenderer headers + return channelHeader.filter(header -> header.headerType != HeaderType.PAGE) + .map(header -> header.json.getObject("banner") + .getArray("thumbnails")) + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElse(List.of()); } @Override @@ -359,9 +340,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index eefc10a94..c88a2918e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -17,6 +17,8 @@ import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -34,6 +36,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.ImageSuffix; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; @@ -43,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -53,6 +57,12 @@ import javax.annotation.Nullable; * {@code youtube.com/watch?v=videoId&list=playlistId} */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private static final List IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of( + // sqdefault and maxresdefault image resolutions are not available on all + // videos, so don't add them in the list of available resolutions + new ImageSuffix("default.jpg", 90, 120, ResolutionLevel.LOW), + new ImageSuffix("mqdefault.jpg", 180, 320, ResolutionLevel.MEDIUM), + new ImageSuffix("hqdefault.jpg", 360, 480, ResolutionLevel.MEDIUM)); /** * YouTube identifies mixes based on this cookie. With this information it can generate @@ -126,18 +136,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { - return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + return getThumbnailsFromPlaylistId(playlistData.getString("playlistId")); } catch (final Exception e) { try { - // Fallback to thumbnail of current video. Always the case for channel mix - return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint") + // Fallback to thumbnail of current video. Always the case for channel mixes + return getThumbnailsFromVideoId(initialData.getObject("currentVideoEndpoint") .getObject("watchEndpoint").getString("videoId")); } catch (final Exception ignored) { } - throw new ParsingException("Could not get playlist thumbnail", e); + throw new ParsingException("Could not get playlist thumbnails", e); } } @@ -153,10 +163,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return "YouTube"; } + @Nonnull @Override - public String getUploaderAvatarUrl() { + public List getUploaderAvatars() { // YouTube mixes are auto-generated by YouTube - return ""; + return List.of(); } @Override @@ -264,14 +275,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } @Nonnull - private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) + private List getThumbnailsFromPlaylistId(@Nonnull final String playlistId) throws ParsingException { - return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); + return getThumbnailsFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); } @Nonnull - private String getThumbnailUrlFromVideoId(final String videoId) { - return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + private List getThumbnailsFromVideoId(@Nonnull final String videoId) { + final String baseUrl = "https://i.ytimg.com/vi/" + videoId + "/"; + return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream() + .map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index bbb5955e0..bb4df7cd2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -3,10 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -15,6 +15,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -33,6 +34,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -160,39 +162,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - String url; + public List getThumbnails() throws ParsingException { + final JsonArray playlistMetadataThumbnailsArray; if (isNewPlaylistInterface) { - url = getPlaylistHeader().getObject("playlistHeaderBanner") + playlistMetadataThumbnailsArray = getPlaylistHeader().getObject("playlistHeaderBanner") .getObject("heroPlaylistThumbnailRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); } else { - url = getPlaylistInfo().getObject("thumbnailRenderer") + playlistMetadataThumbnailsArray = playlistInfo.getObject("thumbnailRenderer") .getObject("playlistVideoThumbnailRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); + } + + if (!isNullOrEmpty(playlistMetadataThumbnailsArray)) { + return getImagesFromThumbnailsArray(playlistMetadataThumbnailsArray); } // This data structure is returned in both layouts - if (isNullOrEmpty(url)) { - url = browseResponse.getObject("microformat") + final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat") .getObject("microformatDataRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); - if (isNullOrEmpty(url)) { - throw new ParsingException("Could not get playlist thumbnail"); - } + if (!isNullOrEmpty(microFormatThumbnailsArray)) { + return getImagesFromThumbnailsArray(microFormatThumbnailsArray); } - return fixThumbnailUrl(url); + throw new ParsingException("Could not get playlist thumbnails"); } @Override @@ -220,23 +218,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { if (isNewPlaylistInterface) { // The new playlist interface doesn't provide an uploader avatar - return ""; + return List.of(); } try { - final String url = getUploaderInfo() - .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); - - return fixThumbnailUrl(url); + return getImagesFromThumbnailsArray(getUploaderInfo().getObject("thumbnail") + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get playlist uploader avatar", e); + throw new ParsingException("Could not get playlist uploader avatars", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 70f6255a8..c78ee5b96 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -30,11 +30,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; @@ -47,6 +48,7 @@ import com.grack.nanojson.JsonWriter; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; @@ -258,23 +260,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { assertPageFetched(); try { - final JsonArray thumbnails = playerResponse - .getObject("videoDetails") + return getImagesFromThumbnailsArray(playerResponse.getObject("videoDetails") .getObject("thumbnail") - .getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails - .getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url"); + throw new ParsingException("Could not get thumbnails"); } - } @Nonnull @@ -552,26 +546,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { assertPageFetched(); - final String url = getVideoSecondaryInfoRenderer() - .getObject("owner") - .getObject("videoOwnerRenderer") - .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + final List imageList = getImagesFromThumbnailsArray( + getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); - if (isNullOrEmpty(url)) { - if (ageLimit == NO_AGE_LIMIT) { - throw new ParsingException("Could not get uploader avatar URL"); - } - - return ""; + if (imageList.isEmpty() && ageLimit == NO_AGE_LIMIT) { + throw new ParsingException("Could not get uploader avatars"); } - return fixThumbnailUrl(url); + return imageList; } @Override From 266cd1f76bf2bf8d6996cac8cecffd5474d7a471 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sun, 24 Jul 2022 19:52:07 +0200 Subject: [PATCH 11/35] [YouTube] Apply changes in YoutubeMusicSearchExtractor and split its InfoItemExtractors into separate classes Splitting YoutubeMusicSearchExtractor's InfoItemExtractors into separate classes (YoutubeMusicSongOrVideoInfoItemExtractor, YoutubeMusicAlbumOrPlaylistInfoItemExtractor and YoutubeMusicArtistInfoItemExtractor) allows to simplify YoutubeMusicSearchExtractor,improves reading and applying changes to InfoItems (no more losing at least quarter of a line due to indentations). These InfoItems, in which the image changes have been applied, don't extend the YouTube ones anymore, as most methods were overridden and the few ones that are not don't apply in YouTube Music items responses, so it was useless to extend them. The code of YoutubeMusicSearchExtractor have been also improved a bit. --- ...MusicAlbumOrPlaylistInfoItemExtractor.java | 157 ++++++++ .../YoutubeMusicArtistInfoItemExtractor.java | 95 +++++ .../YoutubeMusicSearchExtractor.java | 349 ++---------------- ...tubeMusicSongOrVideoInfoItemExtractor.java | 177 +++++++++ 4 files changed, 465 insertions(+), 313 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java new file mode 100644 index 000000000..16619d0a1 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -0,0 +1,157 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_MORE_THAN_100; +import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicAlbumOrPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private final JsonObject albumOrPlaylistInfoItem; + private final JsonArray descriptionElements; + private final String searchType; + + public YoutubeMusicAlbumOrPlaylistInfoItemExtractor(final JsonObject albumOrPlaylistInfoItem, + final JsonArray descriptionElements, + final String searchType) { + this.albumOrPlaylistInfoItem = albumOrPlaylistInfoItem; + this.descriptionElements = descriptionElements; + this.searchType = searchType; + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + albumOrPlaylistInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + + if (!isNullOrEmpty(name)) { + return name; + } + + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String playlistId = albumOrPlaylistInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items") + .getObject(4) + .getObject("toggleMenuServiceItemRenderer") + .getObject("toggledServiceEndpoint") + .getObject("likeEndpoint") + .getObject("target") + .getString("playlistId"); + + if (isNullOrEmpty(playlistId)) { + playlistId = albumOrPlaylistInfoItem.getObject("overlay") + .getObject("musicItemThumbnailOverlayRenderer") + .getObject("content") + .getObject("musicPlayButtonRenderer") + .getObject("playNavigationEndpoint") + .getObject("watchPlaylistEndpoint") + .getString("playlistId"); + } + + if (!isNullOrEmpty(playlistId)) { + return "https://music.youtube.com/playlist?list=" + playlistId; + } + + throw new ParsingException("Could not get URL"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = descriptionElements.getObject(2).getString("text"); + } else { + name = descriptionElements.getObject(0).getString("text"); + } + + if (!isNullOrEmpty(name)) { + return name; + } + + throw new ParsingException("Could not get uploader name"); + } + + @Nullable + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_PLAYLISTS)) { + return null; + } + + final JsonArray items = albumOrPlaylistInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + for (final Object item : items) { + final JsonObject menuNavigationItemRenderer = + ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon") + .getString("iconType", "") + .equals("ARTIST")) { + return getUrlFromNavigationEndpoint( + menuNavigationItemRenderer.getObject("navigationEndpoint")); + } + } + + throw new ParsingException("Could not get uploader URL"); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + + final String count = descriptionElements.getObject(2) + .getString("text"); + + if (!isNullOrEmpty(count)) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + + throw new ParsingException("Could not get stream count"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java new file mode 100644 index 000000000..477eaf709 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -0,0 +1,95 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicArtistInfoItemExtractor implements ChannelInfoItemExtractor { + private final JsonObject artistInfoItem; + + public YoutubeMusicArtistInfoItemExtractor(final JsonObject artistInfoItem) { + this.artistInfoItem = artistInfoItem; + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + artistInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(artistInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint( + artistInfoItem.getObject("navigationEndpoint")); + if (!isNullOrEmpty(url)) { + return url; + } + throw new ParsingException("Could not get URL"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String subscriberCount = getTextFromObject(artistInfoItem.getArray("flexColumns") + .getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(subscriberCount)) { + try { + return Utils.mixedNumberWordToLong(subscriberCount); + } catch (final Parser.RegexException ignored) { + // probably subscriberCount == "No subscribers" or similar + return 0; + } + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public boolean isVerified() { + // An artist on YouTube Music is always verified + return true; + } + + @Nullable + @Override + public String getDescription() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index ae6b10a65..3efe6a74e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -1,9 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -29,19 +27,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -222,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .object("client") .value("clientName", "WEB_REMIX") .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") + .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .array("experimentIds").end() .value("experimentsToken", "") @@ -263,316 +258,44 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } - @SuppressWarnings("MethodLength") private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos) { - final TimeAgoParser timeAgoParser = getTimeAgoParser(); + final String searchType = getLinkHandler().getContentFilters().get(0); + videos.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(item -> item.getObject("musicResponsiveListItemRenderer", null)) + .filter(Objects::nonNull) + .forEachOrdered(infoItem -> { + final String displayPolicy = infoItem.getString( + "musicItemRendererDisplayPolicy", ""); + if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { + // No info about URL available + return; + } - for (final Object item : videos) { - final JsonObject info = ((JsonObject) item) - .getObject("musicResponsiveListItemRenderer", null); - if (info != null) { - final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", - ""); - if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { - continue; // No info about video URL available - } + final JsonArray descriptionElements = infoItem.getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text") + .getArray("runs"); - final JsonObject flexColumnRenderer = info.getArray("flexColumns") - .getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer"); - final JsonArray descriptionElements = flexColumnRenderer.getObject("text") - .getArray("runs"); - final String searchType = getLinkHandler().getContentFilters().get(0); - if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { - collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - final String id = info.getObject("playlistItemData") - .getString("videoId"); - if (!isNullOrEmpty(id)) { - return "https://music.youtube.com/watch?v=" + id; - } - throw new ParsingException("Could not get url"); - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public long getDuration() throws ParsingException { - final String duration = descriptionElements - .getObject(descriptionElements.size() - 1) - .getString("text"); - if (!isNullOrEmpty(duration)) { - return YoutubeParsingHelper.parseDurationString(duration); - } - throw new ParsingException("Could not get duration"); - } - - @Override - public String getUploaderName() throws ParsingException { - final String name = descriptionElements.getObject(0).getString("text"); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get uploader name"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - if (searchType.equals(MUSIC_VIDEOS)) { - final JsonArray items = info.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject( - "menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer - .getObject("navigationEndpoint")); - } - } - - return null; - } else { - final JsonObject navigationEndpointHolder = info - .getArray("flexColumns") - .getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0); - - if (!navigationEndpointHolder.has("navigationEndpoint")) { - return null; - } - - final String url = getUrlFromNavigationEndpoint( - navigationEndpointHolder.getObject("navigationEndpoint")); - - if (!isNullOrEmpty(url)) { - return url; - } - - throw new ParsingException("Could not get uploader URL"); - } - } - - @Override - public String getTextualUploadDate() { - return null; - } - - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - if (searchType.equals(MUSIC_SONGS)) { - return -1; - } - final String viewCount = descriptionElements - .getObject(descriptionElements.size() - 3) - .getString("text"); - if (!isNullOrEmpty(viewCount)) { - try { - return Utils.mixedNumberWordToLong(viewCount); - } catch (final Parser.RegexException e) { - // probably viewCount == "No views" or similar - return 0; - } - } - throw new ParsingException("Could not get view count"); - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } else if (searchType.equals(MUSIC_ARTISTS)) { - collector.commit(new YoutubeChannelInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint(info - .getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - throw new ParsingException("Could not get url"); - } - - @Override - public long getSubscriberCount() throws ParsingException { - final String subscriberCount = getTextFromObject(info - .getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(subscriberCount)) { - try { - return Utils.mixedNumberWordToLong(subscriberCount); - } catch (final Parser.RegexException ignored) { - // probably subscriberCount == "No subscribers" or similar - return 0; - } - } - throw new ParsingException("Could not get subscriber count"); - } - - @Override - public long getStreamCount() { - return -1; - } - - @Override - public String getDescription() { - return null; - } - }); - } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { - collector.commit(new YoutubePlaylistInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String playlistId = info.getObject("menu") - .getObject("menuRenderer") - .getArray("items") - .getObject(4) - .getObject("toggleMenuServiceItemRenderer") - .getObject("toggledServiceEndpoint") - .getObject("likeEndpoint") - .getObject("target") - .getString("playlistId"); - - if (isNullOrEmpty(playlistId)) { - playlistId = info.getObject("overlay") - .getObject("musicItemThumbnailOverlayRenderer") - .getObject("content") - .getObject("musicPlayButtonRenderer") - .getObject("playNavigationEndpoint") - .getObject("watchPlaylistEndpoint") - .getString("playlistId"); - } - if (!isNullOrEmpty(playlistId)) { - return "https://music.youtube.com/playlist?list=" + playlistId; - } - throw new ParsingException("Could not get url"); - } - - @Override - public String getUploaderName() throws ParsingException { - final String name; - if (searchType.equals(MUSIC_ALBUMS)) { - name = descriptionElements.getObject(2).getString("text"); - } else { - name = descriptionElements.getObject(0).getString("text"); - } - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get uploader name"); - } - - @Override - public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) { - return ITEM_COUNT_UNKNOWN; - } - final String count = descriptionElements.getObject(2) - .getString("text"); - if (!isNullOrEmpty(count)) { - if (count.contains("100+")) { - return ITEM_COUNT_MORE_THAN_100; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(count)); - } - } - throw new ParsingException("Could not get count"); - } - }); - } - } - } + switch (searchType) { + case MUSIC_SONGS: + case MUSIC_VIDEOS: + collector.commit(new YoutubeMusicSongOrVideoInfoItemExtractor( + infoItem, descriptionElements, searchType)); + break; + case MUSIC_ARTISTS: + collector.commit(new YoutubeMusicArtistInfoItemExtractor(infoItem)); + break; + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + collector.commit(new YoutubeMusicAlbumOrPlaylistInfoItemExtractor( + infoItem, descriptionElements, searchType)); + break; + } + }); } @Nullable diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java new file mode 100644 index 000000000..11b220288 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -0,0 +1,177 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicSongOrVideoInfoItemExtractor implements StreamInfoItemExtractor { + private final JsonObject songOrVideoInfoItem; + private final JsonArray descriptionElements; + private final String searchType; + + public YoutubeMusicSongOrVideoInfoItemExtractor(final JsonObject songOrVideoInfoItem, + final JsonArray descriptionElements, + final String searchType) { + this.songOrVideoInfoItem = songOrVideoInfoItem; + this.descriptionElements = descriptionElements; + this.searchType = searchType; + } + + + @Override + public String getUrl() throws ParsingException { + final String id = songOrVideoInfoItem.getObject("playlistItemData").getString("videoId"); + if (!isNullOrEmpty(id)) { + return "https://music.youtube.com/watch?v=" + id; + } + throw new ParsingException("Could not get URL"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public StreamType getStreamType() { + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } + + @Override + public long getDuration() throws ParsingException { + final String duration = descriptionElements.getObject(descriptionElements.size() - 1) + .getString("text"); + if (!isNullOrEmpty(duration)) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = descriptionElements.getObject(0).getString("text"); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_VIDEOS)) { + final JsonArray items = songOrVideoInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + for (final Object item : items) { + final JsonObject menuNavigationItemRenderer = + ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon") + .getString("iconType", "") + .equals("ARTIST")) { + return getUrlFromNavigationEndpoint( + menuNavigationItemRenderer.getObject("navigationEndpoint")); + } + } + + return null; + } else { + final JsonObject navigationEndpointHolder = songOrVideoInfoItem.getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text") + .getArray("runs") + .getObject(0); + + if (!navigationEndpointHolder.has("navigationEndpoint")) { + return null; + } + + final String url = getUrlFromNavigationEndpoint( + navigationEndpointHolder.getObject("navigationEndpoint")); + + if (!isNullOrEmpty(url)) { + return url; + } + + throw new ParsingException("Could not get uploader URL"); + } + } + + @Override + public boolean isUploaderVerified() { + // We don't have the ability to know this information on YouTube Music + return false; + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = descriptionElements + .getObject(descriptionElements.size() - 3) + .getString("text"); + if (!isNullOrEmpty(viewCount)) { + try { + return Utils.mixedNumberWordToLong(viewCount); + } catch (final Parser.RegexException e) { + // probably viewCount == "No views" or similar + return 0; + } + } + throw new ParsingException("Could not get view count"); + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + songOrVideoInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } +} From 7f818217d28f3066174bd9cf97088c821938df68 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:40:02 +0200 Subject: [PATCH 12/35] [SoundCloud] Add utility methods to get images from track JSON objects and image URLs These new public and static methods, added in SoundcloudParsingHelper, getAllImagesFromArtworkOrAvatarUrl(String) and getAllImagesFromVisualUrl(String) (which call a common private method, getAllImagesFromImageUrlReturned(String, List, List)), return an unmodifiable list of JPEG images containing almost every image resolution provided by SoundCloud except the original size and the tiny resolution (for artworks and avatars, as the image size is 20x20 for artworks and 18x18 for avatars, so very close to or equal to the t20x20 resolution): - for artworks and avatars: - mini: 16x16; - t20x20: 20x20; - small: 32x32; - badge: 47x47; - t50x50: 50x50; - t60x60: 60x60; - t67x67: 67x67; - large: 100x100; - t120x120: 120x120; - t200x200: 200x200; - t240x240: 240x240; - t250x250: 250x250; - t300x300: 300x300; - t500x500: 500x500. - for visuals/user banners: - t1240x260: 1240x260; - t2480x520: 2480x520. Duplicated code in two methods of SoundcloudParsingHelper (getUsersFromApi(ChannelInfoItemsCollector, String) and getStreamsFromApi(StreamInfoItemsCollector, String, boolean)) has been merged into one common private method, getNextPageUrlFromResponseObject(JsonObject). --- .../soundcloud/SoundcloudParsingHelper.java | 105 +++++++++++++++++- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index a029d85da..bae7c4fe4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import static org.schabi.newpipe.extractor.Image.ResolutionLevel.LOW; +import static org.schabi.newpipe.extractor.Image.ResolutionLevel.MEDIUM; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -9,6 +15,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -20,12 +27,14 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.ImageSuffix; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -35,12 +44,47 @@ import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; import java.util.Map; - -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import java.util.stream.Collectors; public final class SoundcloudParsingHelper { + // CHECKSTYLE:OFF + // From https://web.archive.org/web/20210214185000/https://developers.soundcloud.com/docs/api/reference#tracks + // and researches on images used by the websites + // CHECKSTYLE:ON + /* + SoundCloud avatars and artworks are almost squares + + When we get non-square pictures, all these images variants are still squares, except the + original and the crop versions provides images which are respecting aspect ratios. + The websites only use the square variants. + + t2400x2400 and t3000x3000 variants also exists, but are not returned as several images are + uploaded with a lower size than these variants: in this case, these variants return an upscaled + version of the original image. + */ + private static final List ALBUMS_AND_ARTWORKS_URL_SUFFIXES_AND_RESOLUTIONS = + List.of(new ImageSuffix("mini.jpg", 16, 16, LOW), + new ImageSuffix("t20x20.jpg", 20, 20, LOW), + new ImageSuffix("small.jpg", 32, 32, LOW), + new ImageSuffix("badge.jpg", 47, 47, LOW), + new ImageSuffix("t50x50.jpg", 50, 50, LOW), + new ImageSuffix("t60x60.jpg", 60, 60, LOW), + // Seems to work also on avatars, even if it is written to be not the case in + // the old API docs + new ImageSuffix("t67x67.jpg", 67, 67, LOW), + new ImageSuffix("t80x80.jpg", 80, 80, LOW), + new ImageSuffix("large.jpg", 100, 100, LOW), + new ImageSuffix("t120x120.jpg", 120, 120, LOW), + new ImageSuffix("t200x200.jpg", 200, 200, MEDIUM), + new ImageSuffix("t240x240.jpg", 240, 240, MEDIUM), + new ImageSuffix("t250x250.jpg", 250, 250, MEDIUM), + new ImageSuffix("t300x300.jpg", 300, 300, MEDIUM), + new ImageSuffix("t500x500.jpg", 500, 500, MEDIUM)); + + private static final List VISUALS_URL_SUFFIXES_AND_RESOLUTIONS = + List.of(new ImageSuffix("t1240x260.jpg", 1240, 260, MEDIUM), + new ImageSuffix("t2480x520.jpg", 2480, 520, MEDIUM)); + private static String clientId; public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/"; @@ -366,4 +410,57 @@ public final class SoundcloudParsingHelper { public static String getUploaderName(final JsonObject object) { return object.getObject("user").getString("username", ""); } + + @Nonnull + public static List getAllImagesFromTrackObject(@Nonnull final JsonObject trackObject) + throws ParsingException { + final String artworkUrl = trackObject.getString("artwork_url"); + if (artworkUrl != null) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); + } + final String avatarUrl = trackObject.getObject("user").getString("avatar_url"); + if (avatarUrl != null) { + return getAllImagesFromArtworkOrAvatarUrl(avatarUrl); + } + + throw new ParsingException("Could not get track or track user's thumbnails"); + } + + @Nonnull + public static List getAllImagesFromArtworkOrAvatarUrl( + @Nullable final String originalArtworkOrAvatarUrl) { + if (isNullOrEmpty(originalArtworkOrAvatarUrl)) { + return List.of(); + } + + return getAllImagesFromImageUrlReturned( + // Artwork and avatars are originally returned with the "large" resolution, which + // is 100x100 + originalArtworkOrAvatarUrl.replace("large.jpg", ""), + ALBUMS_AND_ARTWORKS_URL_SUFFIXES_AND_RESOLUTIONS); + } + + @Nonnull + public static List getAllImagesFromVisualUrl( + @Nullable final String originalVisualUrl) { + if (isNullOrEmpty(originalVisualUrl)) { + return List.of(); + } + + return getAllImagesFromImageUrlReturned( + // Images are originally returned with the "original" resolution, which may be + // huge so don't include it for size purposes + originalVisualUrl.replace("original.jpg", ""), + VISUALS_URL_SUFFIXES_AND_RESOLUTIONS); + } + + private static List getAllImagesFromImageUrlReturned( + @Nonnull final String baseImageUrl, + @Nonnull final List imageSuffixes) { + return imageSuffixes.stream() + .map(imageSuffix -> new Image(baseImageUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); + } } From a3a74cd5663dddb3f034469d84d3736cbc153f26 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:58:20 +0200 Subject: [PATCH 13/35] [SoundCloud] Apply changes in InfoItemExtractors and return track user avatars as uploader avatars in SoundcloudStreamInfoItemExtractor --- .../SoundcloudChannelInfoItemExtractor.java | 16 ++++--- .../SoundcloudCommentsInfoItemExtractor.java | 21 ++++++--- .../SoundcloudPlaylistInfoItemExtractor.java | 47 ++++++++++--------- .../SoundcloudStreamInfoItemExtractor.java | 32 +++++++------ 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java index 86b3d48ff..45241f090 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java @@ -1,11 +1,15 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject itemObject; @@ -23,10 +27,10 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac return replaceHttpWithHttps(itemObject.getString("permalink_url")); } + @Nonnull @Override - public String getThumbnailUrl() { - // An avatar URL with a better resolution - return itemObject.getString("avatar_url", "").replace("large.jpg", "crop.jpg"); + public List getThumbnails() { + return getAllImagesFromArtworkOrAvatarUrl(itemObject.getString("avatar_url")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index ec3f353e6..46194a3b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,15 +1,20 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Objects; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; + public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject json; private final String url; @@ -34,9 +39,10 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr return json.getObject("user").getString("username"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return json.getObject("user").getString("avatar_url"); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(json.getObject("user").getString("avatar_url")); } @Override @@ -45,7 +51,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr } @Override - public int getStreamPosition() throws ParsingException { + public int getStreamPosition() { return json.getInt("timestamp") / 1000; // convert milliseconds to seconds } @@ -62,7 +68,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Override @@ -75,8 +81,9 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr return url; } + @Nonnull @Override - public String getThumbnailUrl() { - return json.getObject("user").getString("avatar_url"); + public List getThumbnails() { + return getAllImagesFromArtworkOrAvatarUrl(json.getObject("user").getString("avatar_url")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java index 3f3f3b675..9201b436b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java @@ -1,12 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private static final String USER_KEY = "user"; private static final String AVATAR_URL_KEY = "avatar_url"; @@ -28,36 +33,35 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr return replaceHttpWithHttps(itemObject.getString("permalink_url")); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { // Over-engineering at its finest if (itemObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); - if (!artworkUrl.isEmpty()) { - // An artwork URL with a better resolution - return artworkUrl.replace("large.jpg", "crop.jpg"); + final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY); + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); } } try { - // Look for artwork url inside the track list + // Look for artwork URL inside the track list for (final Object track : itemObject.getArray("tracks")) { final JsonObject trackObject = (JsonObject) track; - // First look for track artwork url + // First look for track artwork URL if (trackObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, ""); - if (!artworkUrl.isEmpty()) { - // An artwork URL with a better resolution - return artworkUrl.replace("large.jpg", "crop.jpg"); + final String artworkUrl = trackObject.getString(ARTWORK_URL_KEY); + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); } } - // Then look for track creator avatar url + // Then look for track creator avatar URL final JsonObject creator = trackObject.getObject(USER_KEY); - final String creatorAvatar = creator.getString(AVATAR_URL_KEY, ""); - if (!creatorAvatar.isEmpty()) { - return creatorAvatar; + final String creatorAvatar = creator.getString(AVATAR_URL_KEY); + if (!isNullOrEmpty(creatorAvatar)) { + return getAllImagesFromArtworkOrAvatarUrl(creatorAvatar); } } } catch (final Exception ignored) { @@ -65,10 +69,11 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr } try { - // Last resort, use user avatar url. If still not found, then throw exception. - return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + // Last resort, use user avatar URL. If still not found, then throw an exception. + return getAllImagesFromArtworkOrAvatarUrl( + itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY)); } catch (final Exception e) { - throw new ParsingException("Failed to extract playlist thumbnail url", e); + throw new ParsingException("Failed to extract playlist thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 3d265e4e4..6fd6232e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -1,20 +1,24 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { - protected final JsonObject itemObject; + private final JsonObject itemObject; public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; @@ -45,10 +49,11 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url")); } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() { - return null; + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl( + itemObject.getObject("user").getString("avatar_url")); } @Override @@ -63,7 +68,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Override @@ -71,13 +76,10 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto return itemObject.getLong("playback_count"); } + @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = itemObject.getString("artwork_url", ""); - if (artworkUrl.isEmpty()) { - artworkUrl = itemObject.getObject("user").getString("avatar_url"); - } - return artworkUrl.replace("large.jpg", "crop.jpg"); + public List getThumbnails() throws ParsingException { + return getAllImagesFromTrackObject(itemObject); } @Override From 31da5beb51120d6d24484d0207eed9779593e81b Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 19:09:17 +0200 Subject: [PATCH 14/35] [SoundCloud] Apply changes in Extractors --- .../SoundcloudChannelExtractor.java | 22 +++++--- .../SoundcloudPlaylistExtractor.java | 55 ++++++++++--------- .../extractors/SoundcloudStreamExtractor.java | 20 +++---- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 143c8bf90..d8cd186d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -1,11 +1,14 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromVisualUrl; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -59,15 +62,19 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return user.getString("username"); } + @Nonnull @Override - public String getAvatarUrl() { - return user.getString("avatar_url"); + public List getAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(user.getString("avatar_url")); } + @Nonnull @Override - public String getBannerUrl() { - return user.getObject("visuals").getArray("visuals").getObject(0) - .getString("visual_url"); + public List getBanners() { + return getAllImagesFromVisualUrl(user.getObject("visuals") + .getArray("visuals") + .getObject(0) + .getString("visual_url")); } @Override @@ -95,9 +102,10 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 88f38b1e1..46bba0a0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -1,9 +1,16 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -24,9 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private static final int STREAMS_PER_REQUESTED_PAGE = 15; @@ -68,30 +72,28 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = playlist.getString("artwork_url"); + public List getThumbnails() { + final String artworkUrl = playlist.getString("artwork_url"); - if (artworkUrl == null) { - // If the thumbnail is null, traverse the items list and get a valid one, - // if it also fails, return null - try { - final InfoItemsPage infoItems = getInitialPage(); - - for (final StreamInfoItem item : infoItems.getItems()) { - artworkUrl = item.getThumbnailUrl(); - if (!isNullOrEmpty(artworkUrl)) { - break; - } - } - } catch (final Exception ignored) { - } - - if (artworkUrl == null) { - return ""; - } + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); } - return artworkUrl.replace("large.jpg", "crop.jpg"); + // If the thumbnail is null or empty, traverse the items list and get a valid one + // If it also fails, return an empty list + try { + final InfoItemsPage infoItems = getInitialPage(); + + for (final StreamInfoItem item : infoItems.getItems()) { + final List thumbnails = item.getThumbnails(); + if (!isNullOrEmpty(thumbnails)) { + return thumbnails; + } + } + } catch (final Exception ignored) { + } + + return List.of(); } @Override @@ -104,9 +106,10 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return SoundcloudParsingHelper.getUploaderName(playlist); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return SoundcloudParsingHelper.getAvatarUrl(playlist); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(getAvatarUrl(playlist)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 22d4d4bea..aeb070153 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -2,6 +2,10 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -11,6 +15,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -96,18 +101,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString( - "created_at"))); + return new DateWrapper(parseDateFrom(track.getString("created_at"))); } @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = track.getString("artwork_url", ""); - if (artworkUrl.isEmpty()) { - artworkUrl = track.getObject("user").getString("avatar_url", ""); - } - return artworkUrl.replace("large.jpg", "crop.jpg"); + public List getThumbnails() throws ParsingException { + return getAllImagesFromTrackObject(track); } @Nonnull @@ -155,8 +155,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getUploaderAvatarUrl() { - return SoundcloudParsingHelper.getAvatarUrl(track); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(getAvatarUrl(track)); } @Override From 81c0d80a54f3b280957ba77908225e2cb91a591c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 27 Jul 2022 19:30:59 +0200 Subject: [PATCH 15/35] [PeerTube] Add utility methods to get avatars and banners of accounts and channels Four new static methods have been added in PeertubeParsingHelper to do so: - two public methods to get the corresponding image type: getAvatarsFromOwnerAccountOrVideoChannelObject(String, JsonObject) and getBannersFromAccountOrVideoChannelObject(String, JsonObject); - two private methods as helper methods: getImagesFromAvatarsOrBanners(String, JsonObject, String, String) and getImagesFromAvatarOrBannerArray(String, JsonArray). --- .../peertube/PeertubeParsingHelper.java | 150 +++++++++++++++++- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index b57aa6b88..ad11e6b39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.peertube; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.Page; @@ -14,12 +16,20 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaSt import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class PeertubeParsingHelper { public static final String START_KEY = "start"; @@ -32,7 +42,7 @@ public final class PeertubeParsingHelper { public static void validate(final JsonObject json) throws ContentNotAvailableException { final String error = json.getString("error"); - if (!Utils.isBlank(error)) { + if (!isBlank(error)) { throw new ContentNotAvailableException(error); } } @@ -53,7 +63,7 @@ public final class PeertubeParsingHelper { } catch (final Parser.RegexException e) { return null; } - if (Utils.isBlank(prevStart)) { + if (isBlank(prevStart)) { return null; } @@ -128,4 +138,138 @@ public final class PeertubeParsingHelper { } } + /** + * Get avatars from a {@code ownerAccount} or a {@code videoChannel} {@link JsonObject}. + * + *

+ * If the {@code avatars} {@link JsonArray} is present and non null or empty, avatars will be + * extracted from this array using {@link #getImagesFromAvatarOrBannerArray(String, JsonArray)}. + *

+ * + *

+ * If that's not the case, an avatar will extracted using the {@code avatar} {@link JsonObject}. + *

+ * + *

+ * Note that only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + public static List getAvatarsFromOwnerAccountOrVideoChannelObject( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject) { + return getImagesFromAvatarsOrBanners(baseUrl, ownerAccountOrVideoChannelObject, + "avatars", "avatar"); + } + + /** + * Get banners from a {@code ownerAccount} or a {@code videoChannel} {@link JsonObject}. + * + *

+ * If the {@code banners} {@link JsonArray} is present and non null or empty, banners will be + * extracted from this array using {@link #getImagesFromAvatarOrBannerArray(String, JsonArray)}. + *

+ * + *

+ * If that's not the case, a banner will extracted using the {@code banner} {@link JsonObject}. + *

+ * + *

+ * Note that only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + public static List getBannersFromAccountOrVideoChannelObject( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject) { + return getImagesFromAvatarsOrBanners(baseUrl, ownerAccountOrVideoChannelObject, + "banners", "banner"); + } + + /** + * Utility method to get avatars and banners from video channels and accounts from given name + * keys. + * + *

+ * Only images for which paths are not null and not empty will be added to the unmodifiable + * {@link Image} list returned and only the width of avatars or banners is provided by the API, + * and so is the only image dimension known. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @param jsonArrayName the key name of the {@link JsonArray} to which + * extract all images available ({@code avatars} or + * {@code banners}) + * @param jsonObjectName the key name of the {@link JsonObject} to which + * extract a single image ({@code avatar} or + * {@code banner}), used as a fallback if the images + * {@link JsonArray} is null or empty + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + private static List getImagesFromAvatarsOrBanners( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject, + @Nonnull final String jsonArrayName, + @Nonnull final String jsonObjectName) { + final JsonArray images = ownerAccountOrVideoChannelObject.getArray(jsonArrayName); + + if (!isNullOrEmpty(images)) { + return getImagesFromAvatarOrBannerArray(baseUrl, images); + } + + final JsonObject image = ownerAccountOrVideoChannelObject.getObject(jsonObjectName); + final String path = image.getString("path"); + if (!isNullOrEmpty(path)) { + return List.of(new Image(baseUrl + path, HEIGHT_UNKNOWN, + image.getInt("width", WIDTH_UNKNOWN), ResolutionLevel.UNKNOWN)); + } + + return List.of(); + } + + /** + * Get {@link Image}s from an {@code avatars} or a {@code banners} {@link JsonArray}. + * + *

+ * Only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + *

+ * Note that only the width of avatars or banners is provided by the API, and so only is the + * only dimension known of images. + *

+ * + * @param baseUrl the base URL of the PeerTube instance from which the + * {@code avatarsOrBannersArray} {@link JsonArray} comes from + * @param avatarsOrBannersArray an {@code avatars} or {@code banners} {@link JsonArray} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + private static List getImagesFromAvatarOrBannerArray( + @Nonnull final String baseUrl, + @Nonnull final JsonArray avatarsOrBannersArray) { + return avatarsOrBannersArray.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(image -> !isNullOrEmpty(image.getString("path"))) + .map(image -> new Image(baseUrl + image.getString("path"), HEIGHT_UNKNOWN, + image.getInt("width", WIDTH_UNKNOWN), ResolutionLevel.UNKNOWN)) + .collect(Collectors.toUnmodifiableList()); + } } From 6f8331524b49a973f449a96cdc943aa0415a2b70 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:43:27 +0100 Subject: [PATCH 16/35] [PeerTube] Add utility method to get thumbnails of playlists and videos This method, getThumbnailsFromPlaylistOrVideoItem, has been added in PeertubeParsingHelper and returns the two image variants for playlists and videos. --- .../peertube/PeertubeParsingHelper.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index ad11e6b39..4e8bd2d35 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -198,6 +199,47 @@ public final class PeertubeParsingHelper { "banners", "banner"); } + /** + * Get thumbnails from a playlist or a video item {@link JsonObject}. + * + *

+ * PeerTube provides two thumbnails in its API: a low one, represented by the value of the + * {@code thumbnailPath} key, and a medium one, represented by the value of the + * {@code previewPath} key. + *

+ * + *

+ * If a value is not null or empty, an {@link Image} will be added to the list returned with + * the URL to the thumbnail ({@code baseUrl + value}), a height and a width unknown and the + * corresponding resolution level (see above). + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param playlistOrVideoItemObject the playlist or the video item {@link JsonObject}, which + * must not be null + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty + */ + @Nonnull + public static List getThumbnailsFromPlaylistOrVideoItem( + @Nonnull final String baseUrl, + @Nonnull final JsonObject playlistOrVideoItemObject) { + final List imageList = new ArrayList<>(2); + + final String thumbnailPath = playlistOrVideoItemObject.getString("thumbnailPath"); + if (!isNullOrEmpty(thumbnailPath)) { + imageList.add(new Image(baseUrl + thumbnailPath, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.LOW)); + } + + final String previewPath = playlistOrVideoItemObject.getString("previewPath"); + if (!isNullOrEmpty(previewPath)) { + imageList.add(new Image(baseUrl + previewPath, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.MEDIUM)); + } + + return Collections.unmodifiableList(imageList); + } + /** * Utility method to get avatars and banners from video channels and accounts from given name * keys. From 0a6011a50ed0dbe4d8acfaeaaf84bb968040e956 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 27 Jul 2022 19:36:44 +0200 Subject: [PATCH 17/35] [PeerTube] Apply changes in InfoItemExtractors Also lower the visibility of attributes of channels and playlists InfoItems to private. --- .../PeertubeChannelInfoItemExtractor.java | 22 +++++++------- .../PeertubeCommentsInfoItemExtractor.java | 29 +++++++------------ .../PeertubePlaylistInfoItemExtractor.java | 16 ++++++---- .../PeertubeStreamInfoItemExtractor.java | 26 +++++++++-------- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java index 0faa25ea8..ac960b07e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java @@ -1,22 +1,24 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import javax.annotation.Nonnull; -import java.util.Comparator; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - final JsonObject item; - final JsonObject uploader; - final String baseUrl; + private final JsonObject item; + private final String baseUrl; + public PeertubeChannelInfoItemExtractor(@Nonnull final JsonObject item, @Nonnull final String baseUrl) { this.item = item; - this.uploader = item.getObject("uploader"); this.baseUrl = baseUrl; } @@ -30,14 +32,10 @@ public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtracto return item.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return item.getArray("avatars").stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .max(Comparator.comparingInt(avatar -> avatar.getInt("width"))) - .map(avatar -> baseUrl + avatar.getString("path")) - .orElse(null); + public List getThumbnails() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 804b23802..f2179cbde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -6,21 +6,24 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Objects; import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -52,15 +55,10 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac return url + "/" + getCommentId(); } + @Nonnull @Override - public String getThumbnailUrl() { - String value; - try { - value = JsonUtils.getString(item, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getThumbnails() { + return getUploaderAvatars(); } @Override @@ -76,7 +74,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(parseDateFrom(textualUploadDate)); } @Override @@ -97,15 +95,10 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac return Objects.toString(item.getLong("id"), null); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - String value; - try { - value = JsonUtils.getString(item, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item.getObject("account")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java index 168872e15..6dbe0272a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java @@ -2,19 +2,22 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nonnull; +import java.util.List; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { +public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - final JsonObject item; - final JsonObject uploader; - final String baseUrl; + private final JsonObject item; + private final JsonObject uploader; + private final String baseUrl; public PeertubePlaylistInfoItemExtractor(@Nonnull final JsonObject item, @Nonnull final String baseUrl) { @@ -33,9 +36,10 @@ public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtrac return item.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + item.getString("thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, item); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index b9c7f4b13..46aae43cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,15 +1,20 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -27,9 +32,10 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(item, "thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, item); } @Override @@ -56,14 +62,10 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor .fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() { - final JsonObject account = item.getObject("account"); - if (account.has("avatar") && !account.isNull("avatar")) { - return baseUrl + account.getObject("avatar").getString("path"); - } - return null; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item.getObject("account")); } @Override @@ -89,7 +91,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return null; } - return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(parseDateFrom(textualUploadDate)); } @Override From 4e6fb368bc49d261a1e3196414449c994249df42 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 29 Jul 2022 12:52:13 +0200 Subject: [PATCH 18/35] [PeerTube] Apply changes in Extractors and remove usages of default avatar picture The default avatar picture was used when no profile picture was found, but it was removed and split in multiple images. Thumbnails' size is not known, as this data is not provided by the API. --- .../extractors/PeertubeAccountExtractor.java | 25 ++++++++------- .../extractors/PeertubeChannelExtractor.java | 32 ++++++++----------- .../extractors/PeertubePlaylistExtractor.java | 21 +++++++----- .../extractors/PeertubeStreamExtractor.java | 28 ++++++---------- 4 files changed, 49 insertions(+), 57 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index eb95dcbe9..8b86a3a44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -22,6 +23,9 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.List; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getBannersFromAccountOrVideoChannelObject; + public class PeertubeAccountExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; @@ -33,20 +37,16 @@ public class PeertubeAccountExtractor extends ChannelExtractor { this.baseUrl = getBaseUrl(); } + @Nonnull @Override - public String getAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json); } + @Nonnull @Override - public String getBannerUrl() { - return null; + public List getBanners() { + return getBannersFromAccountOrVideoChannelObject(baseUrl, json); } @Override @@ -99,9 +99,10 @@ public class PeertubeAccountExtractor extends ChannelExtractor { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e7de3f061..f8594d697 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -20,6 +21,9 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.List; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getBannersFromAccountOrVideoChannelObject; + public class PeertubeChannelExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; @@ -30,20 +34,16 @@ public class PeertubeChannelExtractor extends ChannelExtractor { this.baseUrl = getBaseUrl(); } + @Nonnull @Override - public String getAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json); } + @Nonnull @Override - public String getBannerUrl() { - return null; + public List getBanners() { + return getBannersFromAccountOrVideoChannelObject(baseUrl, json); } @Override @@ -72,15 +72,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor { return JsonUtils.getString(json, "ownerAccount.url"); } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "ownerAccount.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getParentChannelAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject( + baseUrl, json.getObject("ownerAccount")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index a950375f0..e1d05417c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -19,11 +20,14 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.List; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubePlaylistExtractor extends PlaylistExtractor { @@ -36,8 +40,8 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getBaseUrl() + playlistInfo.getString("thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(getBaseUrl(), playlistInfo); } @Override @@ -50,10 +54,11 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { return playlistInfo.getObject("ownerAccount").getString("displayName"); } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { - return getBaseUrl() - + playlistInfo.getObject("ownerAccount").getObject("avatar").getString("path"); + public List getUploaderAvatars() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(getBaseUrl(), + playlistInfo.getObject("ownerAccount")); } @Override @@ -90,9 +95,9 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { - return getBaseUrl() - + playlistInfo.getObject("videoChannel").getObject("avatar").getString("path"); + public List getSubChannelAvatars() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(getBaseUrl(), + playlistInfo.getObject("videoChannel")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 32d3ed4a7..699083df7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -7,7 +9,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -87,8 +89,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "previewPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, json); } @Nonnull @@ -176,14 +178,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getUploaderAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json.getObject("account")); } @Nonnull @@ -200,14 +196,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getSubChannelAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "channel.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getSubChannelAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json.getObject("channel")); } @Nonnull From 4b80d737a4cba22f38767514b1fab6f863be636e Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:02:19 +0200 Subject: [PATCH 19/35] [Bandcamp] Add utility methods to get multiple images Bandcamp images work with image IDs, which provide different resolutions. Images on Bandcamp are not always squares, and some IDs respect aspect ratios where some others not. The extractor will only use the ones which preserve aspect ratio and will not provide original images, for performance and size purposes. Because of this aspect ratio preservation constraint, only one dimension will be known at a time. The image IDs with their respective dimension used are: - 10: 1200w; - 101: 90h; - 170: 422h; - 171: 646h; - 20: 1024w; - 200: 420h; - 201: 280h; - 202: 140h; - 204: 360h; - 205: 240h; - 206: 180h; - 207: 120h; - 43: 100h; - 44: 200h. (Where w represents the width of the image and h the height of the image) Note that these dimensions are theoretical because if the image size is less than the dimensions of the image ID, it will be not upscaled but kept to its original size. All these resolutions are stored in a private static list of ThumbnailSuffixes in BandcampExtractorHelper, in which the methods to get mutliple images have been added: - getImagesFromImageUrl(String): public method to get images from an image URL; - getImagesFromImageId(long, boolean): public method to get images from an image ID; - getImagesFromImageBaseUrl(String): private utility method to get images from the static list of ThumbnailSuffixes from a given image base URL, containing the path to the image, a "a" letter if it comes from an album, its ID and an underscore. Some existing methods have been also edited: - the documentation of getImageUrl(long, boolean) has been changed to reflect the Bandcamp images findings; - getThumbnailUrlFromSearchResult has been renamed to getImagesFromSearchResult, and a documentation has been added to this method. The method replaceHttpWithHttps of the Utils class has been also used in BandcampExtractorHelper instead of doing manually what the method does. --- .../extractors/BandcampExtractorHelper.java | 180 ++++++++++++++++-- 1 file changed, 163 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index aff866053..e7bbac999 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -6,25 +6,81 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.Utils; +import org.schabi.newpipe.extractor.utils.ImageSuffix; -import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.DateTimeException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Collections; +import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public final class BandcampExtractorHelper { + /** + * List of image IDs which preserve aspect ratio with their theoretical dimension known. + * + *

+ * Bandcamp images are not always squares, so images which preserve aspect ratio are only used. + *

+ * + *

+ * One of the direct consequences of this specificity is that only one dimension of images is + * known at time, depending of the image ID. + *

+ * + *

+ * Note also that dimensions are only theoretical because if the image size is less than the + * dimensions of the image ID, it will be not upscaled but kept to its original size. + *

+ * + *

+ * IDs come from the + * GitHub Gist "Bandcamp File Format Parameters" by f2k1de + *

+ */ + private static final List IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of( + // ID | HEIGHT | WIDTH + new ImageSuffix("10.jpg", HEIGHT_UNKNOWN, 1200, ResolutionLevel.HIGH), + new ImageSuffix("101.jpg", 90, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("170.jpg", 422, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + // 180 returns the same image aspect ratio and size as 171 + new ImageSuffix("171.jpg", 646, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("20.jpg", HEIGHT_UNKNOWN, 1024, ResolutionLevel.HIGH), + // 203 returns the same image aspect ratio and size as 200 + new ImageSuffix("200.jpg", 420, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("201.jpg", 280, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("202.jpg", 140, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("204.jpg", 360, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("205.jpg", 240, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("206.jpg", 180, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("207.jpg", 120, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("43.jpg", 100, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("44.jpg", 200, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM)); + + private static final String IMAGE_URL_APPENDIX_AND_EXTENSION_REGEX = "_\\d+\\.\\w+"; + private static final String IMAGES_DOMAIN_AND_PATH = "https://f4.bcbits.com/img/"; + public static final String BASE_URL = "https://bandcamp.com"; public static final String BASE_API_URL = BASE_URL + "/api"; @@ -44,7 +100,7 @@ public final class BandcampExtractorHelper { + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0)) .responseBody(); - return Utils.replaceHttpWithHttps(JsonParser.object().from(jsonString) + return replaceHttpWithHttps(JsonParser.object().from(jsonString) .getString("bandcamp_url")); } catch (final JsonParserException | ReCaptchaException | IOException e) { @@ -76,17 +132,26 @@ public final class BandcampExtractorHelper { } /** - * Generate image url from image ID. - *

- * The appendix "_10" was chosen because it provides images sized 1200x1200. Other integer - * values are possible as well (e.g. 0 is a very large resolution, possibly the original). + * Generate an image url from an image ID. * - * @param id The image ID - * @param album True if this is the cover of an album or track - * @return URL of image with this ID sized 1200x1200 + *

+ * The image ID {@code 10} was chosen because it provides images wide up to 1200px (when + * the original image width is more than or equal this resolution). + *

+ * + *

+ * Other integer values are possible as well (e.g. 0 is a very large resolution, possibly the + * original); see {@link #IMAGE_URL_SUFFIXES_AND_RESOLUTIONS} for more details about image + * resolution IDs. + *

+ * + * @param id the image ID + * @param isAlbum whether the image is the cover of an album or a track + * @return a URL of the image with this ID with a width up to 1200px */ - public static String getImageUrl(final long id, final boolean album) { - return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; + @Nonnull + public static String getImageUrl(final long id, final boolean isAlbum) { + return IMAGES_DOMAIN_AND_PATH + (isAlbum ? 'a' : "") + id + "_10.jpg"; } /** @@ -136,13 +201,94 @@ public final class BandcampExtractorHelper { } } - @Nullable - public static String getThumbnailUrlFromSearchResult(final Element searchResult) { - return searchResult.getElementsByClass("art").stream() + /** + * Get a list of images from a search result {@link Element}. + * + *

+ * This method will call {@link #getImagesFromImageUrl(String)} using the first non null and + * non empty image URL found from the {@code src} attribute of {@code img} HTML elements, or an + * empty string if no valid image URL was found. + *

+ * + * @param searchResult a search result {@link Element} + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where no valid image URL was found + */ + @Nonnull + public static List getImagesFromSearchResult(@Nonnull final Element searchResult) { + return getImagesFromImageUrl(searchResult.getElementsByClass("art") + .stream() .flatMap(element -> element.getElementsByTag("img").stream()) .map(element -> element.attr("src")) - .filter(string -> !string.isEmpty()) + .filter(imageUrl -> !isNullOrEmpty(imageUrl)) .findFirst() - .orElse(null); + .orElse("")); + } + + /** + * Get all images which have resolutions preserving aspect ratio from an image URL. + * + *

+ * This method will remove the image ID and its extension from the end of the URL and then call + * {@link #getImagesFromImageBaseUrl(String)}. + *

+ * + * @param imageUrl the full URL of an image provided by Bandcamp, such as in its HTML code + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where the image URL has been not extracted (and so is null or empty) + */ + @Nonnull + public static List getImagesFromImageUrl(@Nullable final String imageUrl) { + if (isNullOrEmpty(imageUrl)) { + return List.of(); + } + + return getImagesFromImageBaseUrl( + imageUrl.replaceFirst(IMAGE_URL_APPENDIX_AND_EXTENSION_REGEX, "_")); + } + + /** + * Get all images which have resolutions preserving aspect ratio from an image ID. + * + *

+ * This method will call {@link #getImagesFromImageBaseUrl(String)}. + *

+ * + * @param id the id of an image provided by Bandcamp + * @param isAlbum whether the image is the cover of an album + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where the image ID has been not extracted (and so equal to 0) + */ + @Nonnull + public static List getImagesFromImageId(final long id, final boolean isAlbum) { + if (id == 0) { + return List.of(); + } + + return getImagesFromImageBaseUrl(IMAGES_DOMAIN_AND_PATH + (isAlbum ? 'a' : "") + id + "_"); + } + + /** + * Get all images resolutions preserving aspect ratio from a base image URL. + * + *

+ * Base image URLs are images containing the image path, a {@code a} letter if it comes from an + * album, its ID and an underscore. + *

+ * + *

+ * Images resolutions returned are the ones of {@link #IMAGE_URL_SUFFIXES_AND_RESOLUTIONS}. + *

+ * + * @param baseUrl the base URL of the image + * @return an unmodifiable and non-empty list of {@link Image}s + */ + @Nonnull + private static List getImagesFromImageBaseUrl(@Nonnull final String baseUrl) { + return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream() + .map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); } } From 7e01eaac3332567e7c3dc17eb42bb7e508231d40 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:09:09 +0200 Subject: [PATCH 20/35] [Bandcamp] Apply changes in InfoItemExtractors --- .../BandcampAlbumInfoItemExtractor.java | 11 +++++-- .../BandcampChannelInfoItemExtractor.java | 11 +++++-- .../BandcampCommentsInfoItemExtractor.java | 16 +++++++--- .../BandcampPlaylistInfoItemExtractor.java | 9 ++++-- ...campPlaylistInfoItemFeaturedExtractor.java | 17 +++++++--- .../BandcampRadioInfoItemExtractor.java | 20 ++++++------ ...dcampRelatedPlaylistInfoItemExtractor.java | 9 ++++-- ...campDiscographStreamInfoItemExtractor.java | 19 +++++------ ...ndcampPlaylistStreamInfoItemExtractor.java | 32 +++++++++---------- ...BandcampSearchStreamInfoItemExtractor.java | 18 +++++------ 10 files changed, 96 insertions(+), 66 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java index 3f41ea047..5b9b1018f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java @@ -1,10 +1,16 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import java.util.List; + +import javax.annotation.Nonnull; + public class BandcampAlbumInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject albumInfoItem; private final String uploaderUrl; @@ -28,9 +34,10 @@ public class BandcampAlbumInfoItemExtractor implements PlaylistInfoItemExtractor albumInfoItem.getString("item_type")); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getImageUrl(albumInfoItem.getLong("art_id"), true); + public List getThumbnails() throws ParsingException { + return BandcampExtractorHelper.getImagesFromImageId(albumInfoItem.getLong("art_id"), true); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index f89a63405..268494e4a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -3,9 +3,15 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; + public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final Element resultInfo; @@ -26,9 +32,10 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() throws ParsingException { + return getImagesFromSearchResult(searchResult); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java index e65b2cc4d..1a9f691c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java @@ -1,13 +1,17 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; +import java.util.List; + public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject review; @@ -28,9 +32,10 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getUploaderAvatarUrl(); + public List getThumbnails() throws ParsingException { + return getUploaderAvatars(); } @Override @@ -43,8 +48,9 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac return review.getString("name"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return getImageUrl(review.getLong("image_id"), false); + public List getUploaderAvatars() { + return getImagesFromImageId(review.getLong("image_id"), false); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 4bc4a1c65..8c34675f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final Element searchResult; @@ -46,8 +50,9 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() { + return getImagesFromSearchResult(searchResult); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java index 0808e10cb..e7de5691d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -1,9 +1,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoItemExtractor { @@ -40,12 +45,14 @@ public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoIt @Override public String getUrl() { - return featuredStory.getString("item_url").replaceAll("http://", "https://"); + return Utils.replaceHttpWithHttps(featuredStory.getString("item_url")); } + @Nonnull @Override - public String getThumbnailUrl() { - return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) - : getImageUrl(featuredStory.getLong("item_art_id"), true); + public List getThumbnails() { + return featuredStory.has("art_id") + ? getImagesFromImageId(featuredStory.getLong("art_id"), true) + : getImagesFromImageId(featuredStory.getLong("item_art_id"), true); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index be6ba0b7a..023234aa3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -3,15 +3,20 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.parseDate; public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @@ -39,7 +44,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return BandcampExtractorHelper.parseDate(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Override @@ -52,9 +57,10 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { return BASE_URL + "/?show=" + show.getInt("id"); } + @Nonnull @Override - public String getThumbnailUrl() { - return getImageUrl(show.getLong("image_id"), false); + public List getThumbnails() { + return getImagesFromImageId(show.getLong("image_id"), false); } @Override @@ -78,12 +84,6 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { return ""; } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java index 55b319936..64d388528 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java @@ -3,10 +3,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; /** * Extracts recommended albums from tracks' website @@ -28,9 +32,10 @@ public class BandcampRelatedPlaylistInfoItemExtractor implements PlaylistInfoIte return relatedAlbum.getElementsByClass("album-link").attr("abs:href"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return relatedAlbum.getElementsByClass("album-art").attr("src"); + public List getThumbnails() throws ParsingException { + return getImagesFromImageUrl(relatedAlbum.getElementsByClass("album-art").attr("src")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 42b3170b3..cea350460 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -1,10 +1,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { @@ -20,12 +24,6 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf return discograph.getString("band_name"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() { return discograph.getString("title"); @@ -40,11 +38,10 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf ); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getImageUrl( - discograph.getLong("art_id"), true - ); + public List getThumbnails() throws ParsingException { + return getImagesFromImageId(discograph.getLong("art_id"), true); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 1b2e7e886..5c37ec457 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -3,19 +3,21 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; import java.io.IOException; - +import java.util.Collections; +import java.util.List; public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final JsonObject track; - private String substituteCoverUrl; + private List substituteCovers; private final StreamingService service; public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, @@ -24,13 +26,14 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI super(uploaderUrl); this.track = track; this.service = service; + substituteCovers = Collections.emptyList(); } public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, final String uploaderUrl, - final String substituteCoverUrl) { + final List substituteCovers) { this(track, uploaderUrl, (StreamingService) null); - this.substituteCoverUrl = substituteCoverUrl; + this.substituteCovers = substituteCovers; } @Override @@ -56,28 +59,23 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return ""; } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. */ + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - if (substituteCoverUrl != null) { - return substituteCoverUrl; - } else { + public List getThumbnails() throws ParsingException { + if (substituteCovers.isEmpty()) { try { final StreamExtractor extractor = service.getStreamExtractor(getUrl()); extractor.fetchPage(); - return extractor.getThumbnailUrl(); + return extractor.getThumbnails(); } catch (final ExtractionException | IOException e) { - throw new ParsingException("could not download cover art location", e); + throw new ParsingException("Could not download cover art location", e); } } + + return substituteCovers; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 5e585e7e0..18c0c0dcc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -1,10 +1,13 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { @@ -29,12 +32,6 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte } } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); @@ -45,9 +42,10 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() throws ParsingException { + return getImagesFromSearchResult(searchResult); } @Override From 71cda03c4c048209842177983a628d391bf35ac3 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:14:15 +0200 Subject: [PATCH 21/35] [Bandcamp] Apply changes in Extractors --- .../extractors/BandcampChannelExtractor.java | 33 ++++++++++++------- .../extractors/BandcampPlaylistExtractor.java | 21 +++++++----- .../BandcampRadioStreamExtractor.java | 13 +++++--- .../extractors/BandcampStreamExtractor.java | 23 ++++++++----- 4 files changed, 57 insertions(+), 33 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index dc2f98407..ac44cbb8d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,12 +2,18 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getArtistDetails; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; @@ -25,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -38,17 +45,15 @@ public class BandcampChannelExtractor extends ChannelExtractor { super(service, linkHandler); } + @Nonnull @Override - public String getAvatarUrl() { - if (channelInfo.getLong("bio_image_id") == 0) { - return ""; - } - - return BandcampExtractorHelper.getImageUrl(channelInfo.getLong("bio_image_id"), false); + public List getAvatars() { + return getImagesFromImageId(channelInfo.getLong("bio_image_id"), false); } + @Nonnull @Override - public String getBannerUrl() throws ParsingException { + public List getBanners() throws ParsingException { /* * Mobile API does not return the header or not the correct header. * Therefore, we need to query the website @@ -62,8 +67,11 @@ public class BandcampChannelExtractor extends ChannelExtractor { .filter(Objects::nonNull) .flatMap(element -> element.getElementsByTag("img").stream()) .map(element -> element.attr("src")) - .findFirst() - .orElse(""); // no banner available + .filter(url -> !url.isEmpty()) + .map(url -> new Image( + replaceHttpWithHttps(url), HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.UNKNOWN)) + .collect(Collectors.toUnmodifiableList()); } catch (final IOException | ReCaptchaException e) { throw new ParsingException("Could not download artist web site", e); @@ -98,9 +106,10 @@ public class BandcampChannelExtractor extends ChannelExtractor { return null; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return null; + public List getParentChannelAvatars() { + return List.of(); } @Override @@ -156,7 +165,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - channelInfo = BandcampExtractorHelper.getArtistDetails(getId()); + channelInfo = getArtistDetails(getId()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index aa6dd7968..ee5dc9213 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -13,6 +14,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,6 +30,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; import java.util.Objects; +import java.util.List; import javax.annotation.Nonnull; @@ -74,11 +77,11 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { if (albumJson.isNull("art_id")) { - return ""; + return List.of(); } else { - return getImageUrl(albumJson.getLong("art_id"), true); + return getImagesFromImageId(albumJson.getLong("art_id"), true); } } @@ -94,12 +97,14 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { return albumJson.getString("artist"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return document.getElementsByClass("band-photo").stream() + public List getUploaderAvatars() { + return getImagesFromImageUrl(document.getElementsByClass("band-photo") + .stream() .map(element -> element.attr("src")) .findFirst() - .orElse(""); + .orElse("")); } @Override @@ -154,7 +159,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { } else { // Pretend every track has the same cover art as the album collector.commit(new BandcampPlaylistStreamInfoItemExtractor( - track, getUploaderUrl(), getThumbnailUrl())); + track, getUploaderUrl(), getThumbnails())); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 03a9c2222..2b28f03a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -11,6 +12,8 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -97,14 +100,16 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getImageUrl(showInfo.getLong("show_image_id"), false); + public List getThumbnails() throws ParsingException { + return getImagesFromImageId(showInfo.getLong("show_image_id"), false); } @Nonnull @Override - public String getUploaderAvatarUrl() { - return BASE_URL + "/img/buttons/bandcamp-button-circle-whitecolor-512.png"; + public List getUploaderAvatars() { + return Collections.singletonList( + new Image(BASE_URL + "/img/buttons/bandcamp-button-circle-whitecolor-512.png", + 512, 512, ResolutionLevel.MEDIUM)); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index b4e6098ae..dfdbb5696 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -2,8 +2,11 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.parseDate; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParserException; @@ -11,6 +14,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -98,7 +102,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUrl() throws ParsingException { - return albumJson.getString("url").replace("http://", "https://"); + return replaceHttpWithHttps(albumJson.getString("url")); } @Nonnull @@ -116,26 +120,27 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return BandcampExtractorHelper.parseDate(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { if (albumJson.isNull("art_id")) { - return ""; + return List.of(); } - return getImageUrl(albumJson.getLong("art_id"), true); + return getImagesFromImageId(albumJson.getLong("art_id"), true); } @Nonnull @Override - public String getUploaderAvatarUrl() { - return document.getElementsByClass("band-photo").stream() + public List getUploaderAvatars() { + return getImagesFromImageUrl(document.getElementsByClass("band-photo") + .stream() .map(element -> element.attr("src")) .findFirst() - .orElse(""); + .orElse("")); } @Nonnull From 2f40861428eb41e482982ee2e9cc90ecfabc050c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:31:08 +0200 Subject: [PATCH 22/35] [MediaCCC] Add utility methods to get image lists from conference logos and streams These three new methods, added in MediaCCCParsingHelper, getImageListFromImageUrl(String), getThumbnailsFromStreamItem(JsonObject) and getThumbnailsFromLiveStreamItem(JsonObject) (the last two are based on a common method, getThumbnailsFromObject(JsonObject, String, String)), return an empty list if the case no image URL could be extracted. Images returned have their height and width unknown and a resolution level depending on the image key of the JSON API response. --- .../extractors/MediaCCCParsingHelper.java | 108 +++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index ad07c4720..2a4ea6a9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -1,21 +1,33 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.regex.Pattern; +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public final class MediaCCCParsingHelper { - // {conference_slug}/{room_slug} + // conference_slug/room_slug private static final Pattern LIVE_STREAM_ID_PATTERN = Pattern.compile("\\w+/\\w+"); private static JsonArray liveStreams = null; @@ -69,4 +81,98 @@ public final class MediaCCCParsingHelper { } return liveStreams; } + + /** + * Get an {@link Image} list from a given image logo URL. + * + *

+ * If the image URL is null or empty, an empty list is returned; otherwise, a singleton list is + * returned containing an {@link Image} with the image URL with its height, width and + * resolution unknown. + *

+ * + * @param logoImageUrl a logo image URL, which can be null or empty + * @return an unmodifiable list of {@link Image}s, which is always empty or a singleton + */ + @Nonnull + public static List getImageListFromLogoImageUrl(@Nullable final String logoImageUrl) { + if (isNullOrEmpty(logoImageUrl)) { + return List.of(); + } + + return List.of(new Image(logoImageUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.UNKNOWN)); + } + + /** + * Get the {@link Image} list of thumbnails from a given stream item. + * + *

+ * MediaCCC API provides two thumbnails for a stream item: a {@code thumb_url} one, which is + * medium quality and a {@code poster_url} one, which is high quality in most cases. + *

+ * + * @param streamItem a stream JSON item of MediaCCC's API, which must not be null + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + public static List getThumbnailsFromStreamItem(@Nonnull final JsonObject streamItem) { + return getThumbnailsFromObject(streamItem, "thumb_url", "poster_url"); + } + + /** + * Get the {@link Image} list of thumbnails from a given live stream item. + * + *

+ * MediaCCC API provides two URL thumbnails for a livestream item: a {@code thumb} one, + * which should be medium quality and a {@code poster_url} one, which should be high quality. + *

+ * + * @param liveStreamItem a stream JSON item of MediaCCC's API, which must not be null + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + public static List getThumbnailsFromLiveStreamItem( + @Nonnull final JsonObject liveStreamItem) { + return getThumbnailsFromObject(liveStreamItem, "thumb", "poster"); + } + + /** + * Utility method to get an {@link Image} list of thumbnails from a stream or a livestream. + * + *

+ * MediaCCC's API thumbnails come from two elements: a {@code thumb} element, which links to a + * medium thumbnail and a {@code poster} element, which links to a high thumbnail. + *

+ *

+ * Thumbnails are only added if their URLs are not null or empty. + *

+ * + * @param streamOrLivestreamItem a (live)stream JSON item of MediaCCC's API, which must not be + * null + * @param thumbUrlKey the name of the {@code thumb} URL key + * @param posterUrlKey the name of the {@code poster} URL key + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + private static List getThumbnailsFromObject( + @Nonnull final JsonObject streamOrLivestreamItem, + @Nonnull final String thumbUrlKey, + @Nonnull final String posterUrlKey) { + final List imageList = new ArrayList<>(2); + + final String thumbUrl = streamOrLivestreamItem.getString(thumbUrlKey); + if (!isNullOrEmpty(thumbUrl)) { + imageList.add(new Image(thumbUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.MEDIUM)); + } + + final String posterUrl = streamOrLivestreamItem.getString(posterUrlKey); + if (!isNullOrEmpty(posterUrl)) { + imageList.add(new Image(posterUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.HIGH)); + } + + return Collections.unmodifiableList(imageList); + } } From 306068a63b3f4dfa557eaff2b1fd0ddeb7a5ff62 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:39:37 +0200 Subject: [PATCH 23/35] [MediaCCC] Apply changes in InfoItemExtractors --- .../MediaCCCLiveStreamKioskExtractor.java | 16 ++++++++-------- .../extractors/MediaCCCRecentKioskExtractor.java | 16 ++++++++-------- .../MediaCCCConferenceInfoItemExtractor.java | 11 +++++++++-- .../MediaCCCStreamInfoItemExtractor.java | 16 ++++++++-------- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index 2b311fb35..f6c5ac862 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -1,12 +1,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromLiveStreamItem; public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor { @@ -32,9 +37,10 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor return roomInfo.getString("link"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return roomInfo.getString("thumb"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromLiveStreamItem(roomInfo); } @Override @@ -75,12 +81,6 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor return "https://media.ccc.de/c/" + conferenceInfo.getString("slug"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index cef496d9d..df25b28d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; @@ -10,9 +11,13 @@ import org.schabi.newpipe.extractor.stream.StreamType; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.List; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { private final JsonObject event; @@ -31,9 +36,10 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { return event.getString("frontend_link"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return event.getString("thumb_url"); + public List getThumbnails() throws ParsingException { + return getImageListFromLogoImageUrl(event.getString("poster_url")); } @Override @@ -70,12 +76,6 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { .getUrl(); // web URL } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index b69d7a908..3f56aec3a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -2,10 +2,16 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject conference; @@ -43,8 +49,9 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra return conference.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() { - return conference.getString("logo_url"); + public List getThumbnails() { + return getImageListFromLogoImageUrl(conference.getString("logo_url")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 92f0894b9..ec9d00f3a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -1,13 +1,18 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor { private final JsonObject event; @@ -46,12 +51,6 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor return event.getString("conference_url"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; @@ -84,8 +83,9 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor + event.getString("guid"); } + @Nonnull @Override - public String getThumbnailUrl() { - return event.getString("thumb_url"); + public List getThumbnails() { + return getThumbnailsFromStreamItem(event); } } From e16d521b7bbdd3947fe388164ef7f9eb5287466c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:46:54 +0200 Subject: [PATCH 24/35] [MediaCCC] Apply changes in Extractors Also remove usage of the conference logo as the banner of a conference, as it is a logo and not a banner. --- .../MediaCCCConferenceExtractor.java | 19 +++++++++++++------ .../MediaCCCLiveStreamExtractor.java | 6 ++++-- .../extractors/MediaCCCSearchExtractor.java | 8 +++++--- .../extractors/MediaCCCStreamExtractor.java | 14 +++++++++----- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 48d85f7ca..e4574cb88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -21,10 +22,13 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.Medi import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import java.io.IOException; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCConferenceExtractor extends ChannelExtractor { private JsonObject conferenceData; @@ -33,14 +37,16 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { super(service, linkHandler); } + @Nonnull @Override - public String getAvatarUrl() { - return conferenceData.getString("logo_url"); + public List getAvatars() { + return getImageListFromLogoImageUrl(conferenceData.getString("logo_url")); } + @Nonnull @Override - public String getBannerUrl() { - return conferenceData.getString("logo_url"); + public List getBanners() { + return Collections.emptyList(); } @Override @@ -68,9 +74,10 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return Collections.emptyList(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index 9271bbc32..4c3edbd06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -1,11 +1,13 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromLiveStreamItem; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -77,8 +79,8 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return room.getString("thumb"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromLiveStreamItem(room); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index c4501f5c2..e72d26cb2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -9,8 +9,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; @@ -18,7 +20,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; @@ -157,9 +158,10 @@ public class MediaCCCSearchExtractor extends SearchExtractor { return item.getUrl(); } + @Nonnull @Override - public String getThumbnailUrl() { - return item.getThumbnailUrl(); + public List getThumbnails() { + return item.getThumbnails(); } }); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 07dfc870f..9f299a71a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; @@ -8,6 +11,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -51,13 +55,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Nonnull @Override - public String getThumbnailUrl() { - return data.getString("thumb_url"); + public List getThumbnails() { + return getThumbnailsFromStreamItem(data); } @Nonnull @@ -91,8 +95,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getUploaderAvatarUrl() { - return conferenceData.getString("logo_url"); + public List getUploaderAvatars() { + return getImageListFromLogoImageUrl(conferenceData.getString("logo_url")); } @Override From 70fb3aa38e2c15d9c379ce0b4eb84c53dd424cc2 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 1 Aug 2022 21:07:44 +0200 Subject: [PATCH 25/35] Update BaseExtractorTests image methods' name Also suppress unused warnings in BaseStreamExtractorTest, like it is done on other BaseExtractorTests interfaces. --- .../extractor/services/BaseChannelExtractorTest.java | 4 ++-- .../extractor/services/BasePlaylistExtractorTest.java | 6 +++--- .../extractor/services/BaseStreamExtractorTest.java | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java index c776ee514..693b20e18 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java @@ -6,9 +6,9 @@ public interface BaseChannelExtractorTest extends BaseExtractorTest { @Test void testDescription() throws Exception; @Test - void testAvatarUrl() throws Exception; + void testAvatars() throws Exception; @Test - void testBannerUrl() throws Exception; + void testBanners() throws Exception; @Test void testFeedUrl() throws Exception; @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java index 241bbbee5..00478d793 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java @@ -4,13 +4,13 @@ import org.junit.jupiter.api.Test; public interface BasePlaylistExtractorTest extends BaseListExtractorTest { @Test - void testThumbnailUrl() throws Exception; + void testThumbnails() throws Exception; @Test - void testBannerUrl() throws Exception; + void testBanners() throws Exception; @Test void testUploaderName() throws Exception; @Test - void testUploaderAvatarUrl() throws Exception; + void testUploaderAvatars() throws Exception; @Test void testStreamCount() throws Exception; @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index a224a347d..c797220e8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services; import org.junit.jupiter.api.Test; +@SuppressWarnings("unused") public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testStreamType() throws Exception; @@ -10,7 +11,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testUploaderUrl() throws Exception; @Test - void testUploaderAvatarUrl() throws Exception; + void testUploaderAvatars() throws Exception; @Test void testSubscriberCount() throws Exception; @Test @@ -18,9 +19,9 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testSubChannelUrl() throws Exception; @Test - void testSubChannelAvatarUrl() throws Exception; + void testSubChannelAvatars() throws Exception; @Test - void testThumbnailUrl() throws Exception; + void testThumbnails() throws Exception; @Test void testDescription() throws Exception; @Test From 515847285299c69d0d56be23a9c71e0197b284ec Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 1 Aug 2022 21:17:09 +0200 Subject: [PATCH 26/35] Apply changes in DefaultTests and add utility method to test image lists This new method, defaultTestImageList(List thumbnails = item.getThumbnails(); + if (!isNullOrEmpty(thumbnails)) { + defaultTestImageCollection(thumbnails); } assertNotNull(item.getInfoType(), "InfoItem type not set: " + item); assertEquals(expectedService.getServiceId(), item.getServiceId(), "Unexpected item service id"); @@ -44,9 +55,9 @@ public final class DefaultTests { assertExpectedLinkType(expectedService, uploaderUrl, LinkType.CHANNEL); } - final String uploaderAvatarUrl = streamInfoItem.getUploaderAvatarUrl(); - if (!isNullOrEmpty(uploaderAvatarUrl)) { - assertIsSecureUrl(uploaderAvatarUrl); + final List uploaderAvatars = streamInfoItem.getUploaderAvatars(); + if (!isNullOrEmpty(uploaderAvatars)) { + defaultTestImageCollection(uploaderAvatars); } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); @@ -134,4 +145,16 @@ public final class DefaultTests { final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPage); defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors()); } + + public static void defaultTestImageCollection( + @Nullable final Collection imageCollection) { + assertNotNull(imageCollection); + imageCollection.forEach(image -> { + assertIsSecureUrl(image.getUrl()); + assertGreaterOrEqual(Image.HEIGHT_UNKNOWN, image.getHeight(), + "Unexpected image height: " + image.getHeight()); + assertGreaterOrEqual(Image.WIDTH_UNKNOWN, image.getWidth(), + "Unexpected image width: " + image.getWidth()); + }); + } } From 434e88570866a36ab5e767f32ba74e9b43aa36b7 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 13:03:50 +0200 Subject: [PATCH 27/35] Add utility methods in ExtractorAsserts to check whether a collection is empty and to test image collections Two new methods have been added in ExtractorAsserts to check if a collection is empty: - assertNotEmpty(String, Collection), checking: - the non nullity of the collection; - its non emptiness (if that's not case, an exception will be thrown using the provided message). - assertNotEmpty(Collection), calling assertNotEmpty(String, Collection) with null as the value of the string argument. A new one has been added to this assertion class to check the contrary: assertEmpty(Collection), checking emptiness of the collection only if it is not null. Three new methods have been added in ExtractorAsserts as utility test methods for image collections: - assertContainsImageUrlInImageCollection(String, Collection), checking that: - the provided URL and image collection are not null; - the image collection contains at least one image which has the provided string value as its URL (which is a string) property. - assertContainsOnlyEquivalentImages(Collection, Collection), checking that: - both collections are not null; - they have the same size; - each image of the first collection has its equivalent in the second one. This means that the properties of an image in the first collection must be equal in an image of the second one. - assertNotOnlyContainsEquivalentImages(Collection, Collection), checking that: - both collections are not null; - one of the following conditions is met: - they have different sizes; - an image of the first collection has not its equivalent in the second one. This means that the properties of an image in the first collection must be not equal in an image of the second one. These methods will be used by services extractors tests (and default ones) to test image collections. --- .../newpipe/extractor/ExtractorAsserts.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index b47f6d5ee..d66e433ec 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -4,6 +4,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; @@ -60,6 +61,16 @@ public class ExtractorAsserts { assertFalse(stringToCheck.isEmpty(), message); } + public static void assertNotEmpty(@Nullable final Collection collectionToCheck) { + assertNotEmpty(null, collectionToCheck); + } + + public static void assertNotEmpty(@Nullable final String message, + @Nullable final Collection collectionToCheck) { + assertNotNull(collectionToCheck); + assertFalse(collectionToCheck.isEmpty(), message); + } + public static void assertEmpty(String stringToCheck) { assertEmpty(null, stringToCheck); } @@ -70,6 +81,12 @@ public class ExtractorAsserts { } } + public static void assertEmpty(@Nullable final Collection collectionToCheck) { + if (collectionToCheck != null) { + assertTrue(collectionToCheck.isEmpty()); + } + } + public static void assertNotBlank(String stringToCheck) { assertNotBlank(stringToCheck, null); } @@ -160,4 +177,50 @@ public class ExtractorAsserts { .forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab), "Missing " + expectedTab + " tab (got " + tabSet + ")")); } + + public static void assertContainsImageUrlInImageCollection( + @Nullable final String exceptedImageUrlContained, + @Nullable final Collection imageCollection) { + assertNotNull(exceptedImageUrlContained, "exceptedImageUrlContained is null"); + assertNotNull(imageCollection, "imageCollection is null"); + assertTrue(imageCollection.stream().anyMatch(image -> + image.getUrl().equals(exceptedImageUrlContained))); + } + + public static void assertContainsOnlyEquivalentImages( + @Nullable final Collection firstImageCollection, + @Nullable final Collection secondImageCollection) { + assertNotNull(firstImageCollection); + assertNotNull(secondImageCollection); + assertEquals(firstImageCollection.size(), secondImageCollection.size()); + + firstImageCollection.forEach(exceptedImage -> + assertTrue(secondImageCollection.stream().anyMatch(image -> + exceptedImage.getUrl().equals(image.getUrl()) + && exceptedImage.getHeight() == image.getHeight() + && exceptedImage.getWidth() == image.getWidth()))); + } + + public static void assertNotOnlyContainsEquivalentImages( + @Nullable final Collection firstImageCollection, + @Nullable final Collection secondImageCollection) { + assertNotNull(firstImageCollection); + assertNotNull(secondImageCollection); + + if (secondImageCollection.size() != firstImageCollection.size()) { + return; + } + + for (final Image unexpectedImage : firstImageCollection) { + for (final Image image : secondImageCollection) { + if (!image.getUrl().equals(unexpectedImage.getUrl()) + || image.getHeight() != unexpectedImage.getHeight() + || image.getWidth() != unexpectedImage.getWidth()) { + return; + } + } + } + + throw new AssertionError("All excepted images have an equivalent in the image list"); + } } From d381f3b70b20b5a50ce28e744d37980ff0ed3136 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:01:38 +0200 Subject: [PATCH 28/35] Update avatar, banners and thumbnail methods' name and apply changes in DefaultStreamExtractorTest --- .../services/DefaultStreamExtractorTest.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index d9b4e6cde..fab5c8f3a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -29,10 +29,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; @@ -98,8 +100,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Wed, 3 Aug 2022 16:06:21 +0200 Subject: [PATCH 29/35] [YouTube] Add utility test method to test images in YoutubeTestsUtils This method, testImages(Collection), will use first the default image collection test in DefaultTests and then will check that each image URL contains the string yt. The JavaDoc of the class has been also updated to reflect the changes made in it (it is now more general). --- .../services/youtube/YoutubeTestsUtils.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index a98039873..fec933c00 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -1,11 +1,16 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ExtractorAsserts; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import javax.annotation.Nullable; +import java.util.Collection; import java.util.Random; /** - * Utility class for keeping YouTube tests stateless. + * Utility class for YouTube tests. */ public final class YoutubeTestsUtils { private YoutubeTestsUtils() { @@ -26,4 +31,20 @@ public final class YoutubeTestsUtils { YoutubeParsingHelper.setNumberGenerator(new Random(1)); YoutubeStreamExtractor.resetDeobfuscationCode(); } + + /** + * Test that YouTube images of a {@link Collection} respect + * {@link DefaultTests#defaultTestImageCollection(Collection) default requirements} and contain + * the string {@code yt} in their URL. + * + * @param images a YouTube {@link Image} {@link Collection} + */ + public static void testImages(@Nullable final Collection images) { + DefaultTests.defaultTestImageCollection(images); + // Disable NPE warning because if the collection is null, an AssertionError would be thrown + // by DefaultTests.defaultTestImageCollection + //noinspection DataFlowIssue + images.forEach(image -> + ExtractorAsserts.assertContains("yt", image.getUrl())); + } } From 93a210394d1588173b799f7f403f2ddd700c06ab Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:41:03 +0200 Subject: [PATCH 30/35] [YouTube] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and use final in some places where it was possible. --- .../youtube/YoutubeChannelExtractorTest.java | 102 ++++++---------- .../youtube/YoutubeCommentsExtractorTest.java | 44 +++---- .../YoutubeMixPlaylistExtractorTest.java | 51 ++++---- .../youtube/YoutubePlaylistExtractorTest.java | 110 +++++++----------- .../search/YoutubeSearchExtractorTest.java | 23 ++-- .../YoutubeStreamExtractorDefaultTest.java | 4 +- .../YoutubeStreamExtractorRelatedMixTest.java | 5 +- 7 files changed, 143 insertions(+), 196 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 1c2289e4b..856b331ba 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -203,17 +202,13 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -226,6 +221,7 @@ public class YoutubeChannelExtractorTest { ExtractorAsserts.assertGreaterOrEqual(4_900_000, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertTrue(extractor.isVerified()); @@ -248,7 +244,7 @@ public class YoutubeChannelExtractorTest { } } - // Youtube RED/Premium ad blocking test + // YouTube RED/Premium ad blocking test public static class VSauce implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -300,17 +296,13 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -400,17 +392,13 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -524,17 +512,13 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -621,17 +605,13 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -689,7 +669,7 @@ public class YoutubeChannelExtractorTest { @Test public void testName() throws Exception { - assertEquals(extractor.getName(), "Coachella"); + assertEquals("Coachella", extractor.getName()); } @Test @@ -718,16 +698,14 @@ public class YoutubeChannelExtractorTest { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - // CarouselHeaderRenders do not contain a banner - assertNull(extractor.getBannerUrl()); + public void testBanners() { + // A CarouselHeaderRenderer doesn't contain a banner + assertEmpty(extractor.getBanners()); } @Test @@ -795,17 +773,15 @@ public class YoutubeChannelExtractorTest { @Test @Override - public void testAvatarUrl() throws Exception { - final String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test @Override - public void testBannerUrl() throws Exception { + public void testBanners() throws Exception { // Banners cannot be extracted from age-restricted channels - assertTrue(isNullOrEmpty(extractor.getBannerUrl())); + assertEmpty(extractor.getBanners()); } @Test @@ -913,18 +889,14 @@ public class YoutubeChannelExtractorTest { @Test @Override - public void testAvatarUrl() throws Exception { - final String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test @Override - public void testBannerUrl() throws Exception { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index b05502a8a..611c6b457 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -49,7 +49,7 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetComments() throws IOException, ExtractionException { + void testGetComments() throws IOException, ExtractionException { assertTrue(getCommentsHelper(extractor)); } @@ -66,11 +66,11 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { assertTrue(getCommentsFromCommentsInfoHelper(url)); } - private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { + private boolean getCommentsFromCommentsInfoHelper(final String url) throws IOException, ExtractionException { final CommentsInfo commentsInfo = CommentsInfo.getInfo(url); assertEquals("Comments", commentsInfo.getName()); @@ -87,21 +87,21 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); assertTrue(extractor.getCommentsCount() > 5); // at least 5 comments DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getCommentText().getContent())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); } @@ -138,19 +138,19 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text @@ -177,22 +177,22 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); boolean heartedByUploader = false; - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); assertFalse(Utils.isBlank(c.getCommentText().getContent())); @@ -219,20 +219,20 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); assertFalse(Utils.isBlank(c.getCommentText().getContent())); @@ -260,7 +260,7 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsFirst() throws IOException, ExtractionException { + void testGetCommentsFirst() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); @@ -293,7 +293,7 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsFirst() throws IOException, ExtractionException { + void testGetCommentsFirst() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); @@ -319,7 +319,7 @@ public class YoutubeCommentsExtractorTest { } @Test - public void testGetCommentsFirstReplies() throws IOException, ExtractionException { + void testGetCommentsFirstReplies() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index fc765bcde..3913ca2a2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; @@ -69,11 +68,10 @@ public class YoutubeMixPlaylistExtractorTest { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -158,11 +156,10 @@ public class YoutubeMixPlaylistExtractorTest { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -248,10 +245,10 @@ public class YoutubeMixPlaylistExtractorTest { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID)); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -366,10 +363,10 @@ public class YoutubeMixPlaylistExtractorTest { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID_OF_CHANNEL, thumbnail.getUrl())); } @Test @@ -433,11 +430,10 @@ public class YoutubeMixPlaylistExtractorTest { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -523,10 +519,9 @@ public class YoutubeMixPlaylistExtractorTest { @Test void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 1e2a3be1d..5ccec5afb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; @@ -121,22 +120,17 @@ public class YoutubePlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - } - - @Disabled - @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } @Test - public void testUploaderUrl() throws Exception { + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); + } + + @Test + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); } @@ -147,9 +141,8 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -157,6 +150,7 @@ public class YoutubePlaylistExtractorTest { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { assertFalse(extractor.isUploaderVerified()); @@ -191,7 +185,7 @@ public class YoutubePlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testGetPageInNewExtractor() throws Exception { + void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); } @@ -251,22 +245,17 @@ public class YoutubePlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - } - - @Disabled - @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } @Test - public void testUploaderUrl() throws Exception { + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); + } + + @Test + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl()); } @@ -276,9 +265,8 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -286,9 +274,10 @@ public class YoutubePlaylistExtractorTest { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { - assertTrue(extractor.isUploaderVerified()); + assertFalse(extractor.isUploaderVerified()); } @Test @@ -364,22 +353,17 @@ public class YoutubePlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - } - - @Disabled - @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } @Test - public void testUploaderUrl() throws Exception { + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); + } + + @Test + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl()); } @@ -390,9 +374,8 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -400,9 +383,10 @@ public class YoutubePlaylistExtractorTest { ExtractorAsserts.assertGreater(40, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { - assertTrue(extractor.isUploaderVerified()); + assertFalse(extractor.isUploaderVerified()); } @Test @@ -480,18 +464,14 @@ public class YoutubePlaylistExtractorTest { @Test @Override - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } @Test @Override - public void testBannerUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -501,10 +481,8 @@ public class YoutubePlaylistExtractorTest { } @Test - @Override - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -540,7 +518,7 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testNoContinuations() throws Exception { + void testNoContinuations() throws Exception { final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO"); @@ -550,7 +528,7 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testOnlySingleContinuation() throws Exception { + void testOnlySingleContinuation() throws Exception { final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 3c9f310fc..33f2fe43d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -234,9 +234,9 @@ public class YoutubeSearchExtractorTest { } } - public static class PagingTest { + static class PagingTest { @Test - public void duplicatedItemsCheck() throws Exception { + void duplicatedItemsCheck() throws Exception { YoutubeTestsUtils.ensureStateless(); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "paging")); final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), ""); @@ -275,7 +275,7 @@ public class YoutubeSearchExtractorTest { )); } // testMoreRelatedItems is broken because a video has no duration shown - @Override public void testMoreRelatedItems() { } + @Test @Override public void testMoreRelatedItems() { } @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return YouTube; } @Override public String expectedName() { return QUERY; } @@ -307,7 +307,7 @@ public class YoutubeSearchExtractorTest { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } @Test - public void testAtLeastOneVerified() throws IOException, ExtractionException { + void testAtLeastOneVerified() throws IOException, ExtractionException { final List items = extractor.getInitialPage().getItems(); boolean verified = false; for (InfoItem item : items) { @@ -344,11 +344,14 @@ public class YoutubeSearchExtractorTest { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Test - public void testUploaderAvatar() throws IOException, ExtractionException { - final List items = extractor.getInitialPage().getItems(); - for (final InfoItem item : items) { - assertNotNull(((StreamInfoItem) item).getUploaderAvatarUrl()); - } + void testUploaderAvatars() throws IOException, ExtractionException { + extractor.getInitialPage() + .getItems() + .stream() + .filter(StreamInfoItem.class::isInstance) + .map(StreamInfoItem.class::cast) + .forEach(streamInfoItem -> + YoutubeTestsUtils.testImages(streamInfoItem.getUploaderAvatars())); } } @@ -375,7 +378,7 @@ public class YoutubeSearchExtractorTest { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Test - public void testVideoDescription() throws IOException, ExtractionException { + void testVideoDescription() throws IOException, ExtractionException { final List items = extractor.getInitialPage().getItems(); assertNotNull(((StreamInfoItem) items.get(0)).getShortDescription()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index fff75450c..22b5f6f99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -491,8 +491,8 @@ public class YoutubeStreamExtractorDefaultTest { @Test @Override - public void testUploaderAvatarUrl() { - assertThrows(ParsingException.class, () -> extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + assertThrows(ParsingException.class, () -> extractor.getUploaderAvatars()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java index cd7feb352..690647184 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.stream.YoutubeStreamExtractorDefaultTest.YOUTUBE_LICENCE; @@ -109,7 +108,7 @@ public class YoutubeStreamExtractorRelatedMixTest extends DefaultStreamExtractor assertContains(URL, streamMix.getUrl()); assertContains("list=RD" + ID, streamMix.getUrl()); assertEquals("Mix – " + TITLE, streamMix.getName()); - assertIsSecureUrl(streamMix.getThumbnailUrl()); + YoutubeTestsUtils.testImages(streamMix.getThumbnails()); final List musicMixes = playlists.stream() .filter(item -> item.getPlaylistType().equals(PlaylistType.MIX_MUSIC)) @@ -121,6 +120,6 @@ public class YoutubeStreamExtractorRelatedMixTest extends DefaultStreamExtractor assertEquals(YouTube.getServiceId(), musicMix.getServiceId()); assertContains("list=RDCLAK", musicMix.getUrl()); assertEquals("Hip Hop Essentials", musicMix.getName()); - assertIsSecureUrl(musicMix.getThumbnailUrl()); + YoutubeTestsUtils.testImages(musicMix.getThumbnails()); } } From 1d72bac53d4b7a30275ccc479eb0c14f25a4efb9 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:42:49 +0200 Subject: [PATCH 31/35] [SoundCloud] Apply changes in extractor tests --- .../SoundcloudChannelExtractorTest.java | 22 +++---- .../SoundcloudPlaylistExtractorTest.java | 59 +++++++++++-------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index bd94dfd26..7a6e2f1e3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -9,11 +9,13 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link SoundcloudChannelExtractor} @@ -69,13 +71,13 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testAvatarUrl() { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + public void testBanners() { + defaultTestImageCollection(extractor.getBanners()); } @Test @@ -157,13 +159,13 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testAvatarUrl() { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + public void testBanners() { + defaultTestImageCollection(extractor.getBanners()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 1b8930dc6..38fe024e9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -13,11 +13,18 @@ import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.services.DefaultTests.*; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; /** * Test for {@link PlaylistExtractor} @@ -82,14 +89,14 @@ public class SoundcloudPlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -105,8 +112,8 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -179,14 +186,14 @@ public class SoundcloudPlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -202,8 +209,8 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -291,14 +298,14 @@ public class SoundcloudPlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -314,8 +321,8 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -395,14 +402,14 @@ public class SoundcloudPlaylistExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -418,8 +425,8 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test From ba5315c72dcdfc54839cbecd321310bb8d005117 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:51:46 +0200 Subject: [PATCH 32/35] [PeerTube] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and improve some code. --- .../PeertubeAccountExtractorTest.java | 23 ++++---- .../PeertubeChannelExtractorTest.java | 52 ++++++++++--------- .../PeertubeCommentsExtractorTest.java | 5 +- .../PeertubePlaylistExtractorTest.java | 21 +++----- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index 63397c6ba..869ba31ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -15,9 +15,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link PeertubeAccountExtractor} @@ -76,13 +77,13 @@ public class PeertubeAccountExtractorTest { } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -95,6 +96,7 @@ public class PeertubeAccountExtractorTest { ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); @@ -160,18 +162,18 @@ public class PeertubeAccountExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -184,6 +186,7 @@ public class PeertubeAccountExtractorTest { ExtractorAsserts.assertGreaterOrEqual(100, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 72ce04839..5b48849f8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,11 +12,12 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannel import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link PeertubeChannelExtractor} @@ -71,33 +71,33 @@ public class PeertubeChannelExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testParentChannelName() throws ParsingException { + void testParentChannelName() throws ParsingException { assertEquals("lqdn", extractor.getParentChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { + void testParentChannelUrl() throws ParsingException { assertEquals("https://video.lqdn.fr/accounts/lqdn", extractor.getParentChannelUrl()); } @Test - public void testParentChannelAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); + void testParentChannelAvatarUrl() { + defaultTestImageCollection(extractor.getParentChannelAvatars()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -106,10 +106,11 @@ public class PeertubeChannelExtractorTest { } @Test - public void testSubscriberCount() throws ParsingException { - ExtractorAsserts.assertGreaterOrEqual(230, extractor.getSubscriberCount()); + public void testSubscriberCount() { + assertGreaterOrEqual(230, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); @@ -176,33 +177,33 @@ public class PeertubeChannelExtractorTest { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testParentChannelName() throws ParsingException { + void testParentChannelName() throws ParsingException { assertEquals("nathan", extractor.getParentChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { + void testParentChannelUrl() throws ParsingException { assertEquals("https://skeptikon.fr/accounts/nathan", extractor.getParentChannelUrl()); } @Test - public void testParentChannelAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); + void testParentChannelAvatars() { + defaultTestImageCollection(extractor.getParentChannelAvatars()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() throws ParsingException { + assertEmpty(extractor.getBanners()); } @Test @@ -211,10 +212,11 @@ public class PeertubeChannelExtractorTest { } @Test - public void testSubscriberCount() throws ParsingException { - ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); + public void testSubscriberCount() { + assertGreaterOrEqual(700, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index 353e00482..ee2303706 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -18,6 +18,7 @@ import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; public class PeertubeCommentsExtractorTest { public static class Default { @@ -73,12 +74,12 @@ public class PeertubeCommentsExtractorTest { .forEach(commentsInfoItem -> { assertFalse(Utils.isBlank(commentsInfoItem.getUploaderUrl())); assertFalse(Utils.isBlank(commentsInfoItem.getUploaderName())); - assertFalse(Utils.isBlank(commentsInfoItem.getUploaderAvatarUrl())); + defaultTestImageCollection(commentsInfoItem.getUploaderAvatars()); assertFalse(Utils.isBlank(commentsInfoItem.getCommentId())); assertFalse(Utils.isBlank(commentsInfoItem.getCommentText().getContent())); assertFalse(Utils.isBlank(commentsInfoItem.getName())); assertFalse(Utils.isBlank(commentsInfoItem.getTextualUploadDate())); - assertFalse(Utils.isBlank(commentsInfoItem.getThumbnailUrl())); + defaultTestImageCollection(commentsInfoItem.getThumbnails()); assertFalse(Utils.isBlank(commentsInfoItem.getUrl())); assertEquals(-1, commentsInfoItem.getLikeCount()); assertTrue(Utils.isBlank(commentsInfoItem.getTextualLikeCount())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java index 37853305a..4c64314cd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -2,9 +2,9 @@ package org.schabi.newpipe.extractor.services.peertube; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; @@ -31,11 +31,8 @@ public class PeertubePlaylistExtractorTest { } @Test - @Disabled("URL changes with every request") - void testGetThumbnailUrl() throws ParsingException { - assertEquals( - "https://framatube.org/static/thumbnails/playlist-96b0ee2b-a5a7-4794-8769-58d8ccb79ab7.jpg", - extractor.getThumbnailUrl()); + void testGetThumbnails() throws ParsingException { + defaultTestImageCollection(extractor.getThumbnails()); } @Test @@ -44,10 +41,8 @@ public class PeertubePlaylistExtractorTest { } @Test - void testGetUploaderAvatarUrl() throws ParsingException { - assertEquals( - "https://framatube.org/lazy-static/avatars/c6801ff9-cb49-42e6-b2db-3db623248115.jpg", - extractor.getUploaderAvatarUrl()); + void testGetUploaderAvatars() throws ParsingException { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -76,10 +71,8 @@ public class PeertubePlaylistExtractorTest { } @Test - void testGetSubChannelAvatarUrl() throws ParsingException { - assertEquals( - "https://framatube.org/lazy-static/avatars/e801ccce-8694-4309-b0ab-e6f0e552ef77.png", - extractor.getSubChannelAvatarUrl()); + void testGetSubChannelAvatars() throws ParsingException { + defaultTestImageCollection(extractor.getSubChannelAvatars()); } } } From 2578f220540b04ed68ef1f971f7bf75d5e48e8c4 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 17:47:58 +0200 Subject: [PATCH 33/35] [Bandcamp] Add utility test method to test images This method, testImages(Collection), will use first the default image collection test in DefaultTests and then will check that each image URL contains f4.bcbits.com/img and ends with .jpg or .png. To do so, a new non-instantiable final class has been added: BandcampTestUtils. --- .../services/bandcamp/BandcampTestUtils.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java new file mode 100644 index 000000000..bc17fc515 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java @@ -0,0 +1,34 @@ +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.services.DefaultTests; + +import javax.annotation.Nullable; +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Utility class for Bandcamp tests. + */ +public final class BandcampTestUtils { + private BandcampTestUtils() { + } + + /** + * Test that Bandcamp images of a {@link Collection} respect + * {@link DefaultTests#defaultTestImageCollection(Collection) default requirements}, contain + * the string {@code f4.bcbits.com/img} in their URL and end with {@code .jpg} or {@code .png}. + * + * @param images a Bandcamp {@link Image} {@link Collection} + */ + public static void testImages(@Nullable final Collection images) { + DefaultTests.defaultTestImageCollection(images); + // Disable NPE warning because if the collection is null, an AssertionError would be thrown + // by DefaultTests.defaultTestImageCollection + //noinspection DataFlowIssue + assertTrue(images.stream() + .allMatch(image -> image.getUrl().contains("f4.bcbits.com/img") + && (image.getUrl().endsWith(".jpg") || image.getUrl().endsWith(".png")))); + } +} From 0292c4f3e848726de0bd697eed10cf2af702ba37 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 18:05:44 +0200 Subject: [PATCH 34/35] [Bandcamp] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and use final in some places where it was possible. BandcampChannelExtractorTest.testLength has been removed as the test is always true. --- .../BandcampChannelExtractorTest.java | 23 ++++++-- .../BandcampCommentsExtractorTest.java | 10 ++-- .../BandcampPlaylistExtractorTest.java | 53 +++++++++++-------- .../BandcampRadioStreamExtractorTest.java | 22 +++++--- .../bandcamp/BandcampSearchExtractorTest.java | 14 ++--- .../bandcamp/BandcampStreamExtractorTest.java | 8 ++- 6 files changed, 76 insertions(+), 54 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 066c7a5ae..0fae98b53 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -10,7 +10,10 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -31,51 +34,61 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { assertEquals("making music:)", extractor.getDescription()); } + @Test @Override - public void testAvatarUrl() throws Exception { - assertTrue(extractor.getAvatarUrl().contains("://f4.bcbits.com/"), "unexpected avatar URL"); + public void testAvatars() throws Exception { + BandcampTestUtils.testImages(extractor.getAvatars()); } + @Test @Override - public void testBannerUrl() throws Exception { - assertTrue(extractor.getBannerUrl().contains("://f4.bcbits.com/"), "unexpected banner URL"); + public void testBanners() throws Exception { + BandcampTestUtils.testImages(extractor.getBanners()); } + @Test @Override public void testFeedUrl() throws Exception { assertNull(extractor.getFeedUrl()); } + @Test @Override public void testSubscriberCount() throws Exception { assertEquals(-1, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } + @Test @Override public void testServiceId() { assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); } + @Test @Override public void testName() throws Exception { assertEquals("toupie", extractor.getName()); } + @Test @Override public void testId() throws Exception { assertEquals("2450875064", extractor.getId()); } + @Test @Override public void testUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } + @Test @Override public void testOriginalUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java index 2fd98e255..ed4985296 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java @@ -30,22 +30,22 @@ public class BandcampCommentsExtractorTest { } @Test - public void hasComments() throws IOException, ExtractionException { + void hasComments() throws IOException, ExtractionException { assertTrue(extractor.getInitialPage().getItems().size() >= 3); } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { ListExtractor.InfoItemsPage comments = extractor.getInitialPage(); assertTrue(comments.hasNextPage()); DefaultTests.defaultTestListOfItems(Bandcamp, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + BandcampTestUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentText().getContent())); assertFalse(Utils.isBlank(c.getName())); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + BandcampTestUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertEquals(-1, c.getLikeCount()); assertTrue(Utils.isBlank(c.getTextualLikeCount())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index f6f097978..448cca262 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -19,8 +19,17 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsOnlyEquivalentImages; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotOnlyContainsEquivalentImages; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** @@ -37,7 +46,7 @@ public class BandcampPlaylistExtractorTest { * Test whether playlists contain the correct amount of items */ @Test - public void testCount() throws ExtractionException, IOException { + void testCount() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); extractor.fetchPage(); @@ -48,13 +57,13 @@ public class BandcampPlaylistExtractorTest { * Tests whether different stream thumbnails (track covers) get loaded correctly */ @Test - public void testDifferentTrackCovers() throws ExtractionException, IOException { + void testDifferentTrackCovers() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); extractor.fetchPage(); final List l = extractor.getInitialPage().getItems(); - assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); - assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(0).getThumbnails()); + assertNotOnlyContainsEquivalentImages(extractor.getThumbnails(), l.get(5).getThumbnails()); } /** @@ -62,23 +71,23 @@ public class BandcampPlaylistExtractorTest { */ @Test @Timeout(10) - public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { + void testDifferentTrackCoversDuration() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); extractor.fetchPage(); - /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks - * that has at least one track with a cover art different from the rest. + /* All tracks on this album have the same cover art, but I don't know any albums with more + * than 10 tracks that has at least one track with a cover art different from the rest. */ final List l = extractor.getInitialPage().getItems(); - assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); - assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(0).getThumbnails()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(5).getThumbnails()); } /** * Test playlists with locked content */ @Test - public void testLockedContent() throws ExtractionException, IOException { + void testLockedContent() throws ExtractionException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); assertThrows(ContentNotAvailableException.class, extractor::fetchPage); @@ -88,12 +97,11 @@ public class BandcampPlaylistExtractorTest { * Test playlist with just one track */ @Test - public void testSingleStreamPlaylist() throws ExtractionException, IOException { + void testSingleStreamPlaylist() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); extractor.fetchPage(); assertEquals(1, extractor.getStreamCount()); - } public static class ComingOfAge implements BasePlaylistExtractorTest { @@ -108,17 +116,17 @@ public class BandcampPlaylistExtractorTest { } @Test - public void testThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl().contains("f4.bcbits.com/img")); + public void testThumbnails() throws ParsingException { + BandcampTestUtils.testImages(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { - assertEquals("", extractor.getBannerUrl()); + public void testBanners() throws ParsingException { + assertEmpty(extractor.getBanners()); } @Test - public void testUploaderUrl() throws ParsingException { + void testUploaderUrl() throws ParsingException { assertTrue(extractor.getUploaderUrl().contains("macbenson.bandcamp.com")); } @@ -128,8 +136,8 @@ public class BandcampPlaylistExtractorTest { } @Test - public void testUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl().contains("f4.bcbits.com/img")); + public void testUploaderAvatars() throws ParsingException { + BandcampTestUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -147,13 +155,14 @@ public class BandcampPlaylistExtractorTest { assertContains("all rights reserved", description.getContent()); // license } + @Test @Override public void testUploaderVerified() throws Exception { assertFalse(extractor.isUploaderVerified()); } @Test - public void testInitialPage() throws IOException, ExtractionException { + void testInitialPage() throws IOException, ExtractionException { assertNotNull(extractor.getInitialPage().getItems().get(0)); } @@ -183,7 +192,7 @@ public class BandcampPlaylistExtractorTest { } @Test - public void testNextPageUrl() throws IOException, ExtractionException { + void testNextPageUrl() throws IOException, ExtractionException { assertNull(extractor.getPage(extractor.getInitialPage().getNextPage())); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index e00a61f3b..920c32a7b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -3,12 +3,14 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -36,7 +38,7 @@ public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest } @Test - public void testGettingCorrectStreamExtractor() throws ExtractionException { + void testGettingCorrectStreamExtractor() throws ExtractionException { assertTrue(Bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); assertFalse(Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); @@ -57,15 +59,16 @@ public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest @Override public int expectedStreamSegmentsCount() { return 30; } @Test - public void testGetUploaderUrl() { + void testGetUploaderUrl() { assertThrows(ContentNotSupportedException.class, extractor::getUploaderUrl); } @Test @Override - public void testUploaderUrl() throws Exception { + public void testUploaderUrl() { assertThrows(ContentNotSupportedException.class, super::testUploaderUrl); } + @Override public String expectedUploaderUrl() { return null; } @Override @@ -93,16 +96,19 @@ public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest } @Test - public void testGetThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl().contains("bcbits.com/img")); + void testGetThumbnails() throws ParsingException { + BandcampTestUtils.testImages(extractor.getThumbnails()); } @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl().contains("bandcamp-button")); + void testGetUploaderAvatars() throws ParsingException { + DefaultTests.defaultTestImageCollection(extractor.getUploaderAvatars()); + extractor.getUploaderAvatars().forEach(image -> + ExtractorAsserts.assertContains("bandcamp-button", image.getUrl())); } - @Test public void testGetAudioStreams() throws ExtractionException, IOException { + @Test + void testGetAudioStreams() throws ExtractionException, IOException { assertEquals(1, extractor.getAudioStreams().size()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index d60178bd7..3b4052424 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.bandcamp; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; import org.junit.jupiter.api.BeforeAll; @@ -50,8 +49,7 @@ public class BandcampSearchExtractorTest { // The track by Zach Benson should be the first result, no? assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); assertEquals("Zach Benson", bestFriendsBasement.getUploaderName()); - assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); - assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); + BandcampTestUtils.testImages(bestFriendsBasement.getThumbnails()); assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); } @@ -66,10 +64,8 @@ public class BandcampSearchExtractorTest { // C418's artist profile should be the first result, no? assertEquals("C418", c418.getName()); - assertTrue(c418.getThumbnailUrl().endsWith(".jpg")); - assertTrue(c418.getThumbnailUrl().contains("f4.bcbits.com/img/")); + BandcampTestUtils.testImages(c418.getThumbnails()); assertEquals("https://c418.bandcamp.com", c418.getUrl()); - } /** @@ -82,9 +78,9 @@ public class BandcampSearchExtractorTest { // Minecraft volume alpha should be the first result, no? assertEquals("Minecraft - Volume Alpha", minecraft.getName()); - assertTrue(minecraft.getThumbnailUrl().endsWith(".jpg")); - assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); - assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", + BandcampTestUtils.testImages(minecraft.getThumbnails()); + assertEquals( + "https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); // Verify that playlist tracks counts get extracted correctly diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 3a582db28..38bb44f3c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -20,7 +20,6 @@ import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** @@ -150,13 +149,12 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { } @Test - public void testArtistProfilePicture() throws Exception { - final String url = extractor().getUploaderAvatarUrl(); - assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); + void testArtistProfilePictures() { + BandcampTestUtils.testImages(extractor.getUploaderAvatars()); } @Test - public void testTranslateIdsToUrl() throws ParsingException { + void testTranslateIdsToUrl() throws ParsingException { // To add tests: look at website's source, search for `band_id` and `item_id` assertEquals( "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution", From e8bfd20170581d8ec4722792acca75f37bafcec3 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 18:09:52 +0200 Subject: [PATCH 35/35] [MediaCCC] Apply changes in extractor tests Also remove some public test methods modifiers. --- .../MediaCCCConferenceExtractorTest.java | 29 +++++++++------- .../MediaCCCStreamExtractorTest.java | 33 ++++++++++++------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 976af4e42..86561c971 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConfer import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsImageUrlInImageCollection; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -30,27 +31,29 @@ public class MediaCCCConferenceExtractorTest { } @Test - public void testName() throws Exception { + void testName() throws Exception { assertEquals("FrOSCon 2017", extractor.getName()); } @Test - public void testGetUrl() throws Exception { + void testGetUrl() throws Exception { assertEquals("https://media.ccc.de/c/froscon2017", extractor.getUrl()); } @Test - public void testGetOriginalUrl() throws Exception { + void testGetOriginalUrl() throws Exception { assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); } @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + void testGetThumbnails() { + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/froscon/2017/logo.png", + extractor.getAvatars()); } @Test - public void testGetInitalPage() throws Exception { + void testGetInitalPage() throws Exception { assertEquals(97, tabExtractor.getInitialPage().getItems().size()); } } @@ -70,27 +73,29 @@ public class MediaCCCConferenceExtractorTest { } @Test - public void testName() throws Exception { + void testName() throws Exception { assertEquals("Open Source Conference Albania 2019", extractor.getName()); } @Test - public void testGetUrl() throws Exception { + void testGetUrl() throws Exception { assertEquals("https://media.ccc.de/c/oscal19", extractor.getUrl()); } @Test - public void testGetOriginalUrl() throws Exception { + void testGetOriginalUrl() throws Exception { assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl()); } @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl()); + void testGetThumbnailUrl() { + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", + extractor.getAvatars()); } @Test - public void testGetInitalPage() throws Exception { + void testGetInitalPage() throws Exception { assertTrue(tabExtractor.getInitialPage().getItems().size() >= 21); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index feb0b7c72..b68ce9ff4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -21,6 +21,7 @@ import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsImageUrlInImageCollection; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -66,16 +67,20 @@ public class MediaCCCStreamExtractorTest { @Override @Test - public void testThumbnailUrl() throws Exception { - super.testThumbnailUrl(); - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + public void testThumbnails() throws Exception { + super.testThumbnails(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/gpn/gpn18/105-hd_preview.jpg", + extractor.getThumbnails()); } @Override @Test - public void testUploaderAvatarUrl() throws Exception { - super.testUploaderAvatarUrl(); - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() throws Exception { + super.testUploaderAvatars(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", + extractor.getUploaderAvatars()); } @Override @@ -140,16 +145,20 @@ public class MediaCCCStreamExtractorTest { @Override @Test - public void testThumbnailUrl() throws Exception { - super.testThumbnailUrl(); - assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); + public void testThumbnails() throws Exception { + super.testThumbnails(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/congress/2019/10565-hd_preview.jpg", + extractor.getThumbnails()); } @Override @Test - public void testUploaderAvatarUrl() throws Exception { - super.testUploaderAvatarUrl(); - assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() throws Exception { + super.testUploaderAvatars(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/congress/2019/logo.png", + extractor.getUploaderAvatars()); } @Override