From c47cc54908d14d3faa6326e62cb8c8add9463df2 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 9 Feb 2020 11:59:23 +0100 Subject: [PATCH] Extract metadata for YouTube, SoundCloud & MediaCCC --- .../extractor/localization/DateWrapper.java | 18 +- .../extractor/localization/Localization.java | 35 +++- .../BandcampRadioStreamExtractor.java | 6 + .../extractors/BandcampStreamExtractor.java | 4 +- .../MediaCCCLiveStreamExtractor.java | 4 +- .../extractors/MediaCCCStreamExtractor.java | 17 +- .../extractors/PeertubeStreamExtractor.java | 21 +- .../extractors/SoundcloudStreamExtractor.java | 33 ++- .../extractors/YoutubeStreamExtractor.java | 27 ++- .../extractor/stream/StreamExtractor.java | 11 +- .../newpipe/extractor/stream/StreamInfo.java | 7 +- .../newpipe/extractor/utils/JsonUtils.java | 10 + .../newpipe/extractor/ExtractorAsserts.java | 8 +- .../services/DefaultStreamExtractorTest.java | 2 +- .../MediaCCCStreamExtractorTest.java | 188 +++++++++++++++--- .../peertube/PeertubeStreamExtractorTest.java | 2 - .../SoundcloudStreamExtractorTest.java | 29 ++- ...utubeStreamExtractorAgeRestrictedTest.java | 12 ++ ...utubeStreamExtractorControversialTest.java | 4 + .../YoutubeStreamExtractorDefaultTest.java | 83 +++++++- .../YoutubeStreamExtractorLivestreamTest.java | 14 +- .../YoutubeStreamExtractorUnlistedTest.java | 5 + 22 files changed, 440 insertions(+), 100 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index f86efa9b9..b1424e3b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.localization; -import edu.umd.cs.findbugs.annotations.NonNull; +import javax.annotation.Nonnull; import java.io.Serializable; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -12,14 +12,15 @@ import java.util.GregorianCalendar; * A wrapper class that provides a field to describe if the date/time is precise or just an approximation. */ public class DateWrapper implements Serializable { - @NonNull private final OffsetDateTime offsetDateTime; + @Nonnull + private final OffsetDateTime offsetDateTime; private final boolean isApproximation; /** * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead. */ @Deprecated - public DateWrapper(@NonNull Calendar calendar) { + public DateWrapper(@Nonnull Calendar calendar) { this(calendar, false); } @@ -27,26 +28,25 @@ public class DateWrapper implements Serializable { * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead. */ @Deprecated - public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) { + public DateWrapper(@Nonnull Calendar calendar, boolean isApproximation) { this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation); } - public DateWrapper(@NonNull OffsetDateTime offsetDateTime) { + public DateWrapper(@Nonnull OffsetDateTime offsetDateTime) { this(offsetDateTime, false); } - public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) { + public DateWrapper(@Nonnull OffsetDateTime offsetDateTime, boolean isApproximation) { this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC); this.isApproximation = isApproximation; } /** * @return the wrapped date/time as a {@link Calendar}. - * * @deprecated use {@link #offsetDateTime()} instead. */ @Deprecated - @NonNull + @Nonnull public Calendar date() { return GregorianCalendar.from(offsetDateTime.toZonedDateTime()); } @@ -54,7 +54,7 @@ public class DateWrapper implements Serializable { /** * @return the wrapped date/time. */ - @NonNull + @Nonnull public OffsetDateTime offsetDateTime() { return offsetDateTime; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index a3af889ac..83fc84454 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -1,19 +1,19 @@ package org.schabi.newpipe.extractor.localization; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Objects; +import java.util.*; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); - @Nonnull private final String languageCode; - @Nullable private final String countryCode; + @Nonnull + private final String languageCode; + @Nullable + private final String countryCode; /** * @param localizationCodeList a list of localization code, formatted like {@link #getLocalizationCode()} @@ -100,4 +100,25 @@ public class Localization implements Serializable { result = 31 * result + Objects.hashCode(countryCode); return result; } + + /** + * Converts a three letter language code (ISO 639-2/T) to a Locale + * in the limit of Java Locale class. + * + * @param code a three letter language code + * @return the Locale corresponding + */ + public static Locale getLocaleFromThreeLetterCode(@Nonnull String code) throws ParsingException { + String[] languages = Locale.getISOLanguages(); + Map localeMap = new HashMap<>(languages.length); + for (String language : languages) { + final Locale locale = new Locale(language); + localeMap.put(locale.getISO3Language(), locale); + } + if (localeMap.containsKey(code)) { + return localeMap.get(code); + } else { + throw new ParsingException("Could not get Locale from this three letter language code" + code); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 7d2543c8a..d1ef8d862 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -140,4 +140,10 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { public List getTags() { return Collections.emptyList(); } + + @Nonnull + @Override + public Privacy getPrivacy() { + return Privacy.PUBLIC; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 344663525..7a67c07d4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -262,8 +262,8 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getPrivacy() { - return ""; + public Privacy getPrivacy() { + return Privacy.PUBLIC; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index 77ef68934..889c3c9f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -257,8 +257,8 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getPrivacy() { - return "Public"; + public Privacy getPrivacy() { + return Privacy.PUBLIC; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 1ae8508aa..a9c31c72d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,14 +12,19 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class MediaCCCStreamExtractor extends StreamExtractor { private JsonObject data; @@ -256,8 +261,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getPrivacy() { - return ""; + public Privacy getPrivacy() { + return Privacy.PUBLIC; } @Nonnull @@ -273,14 +278,14 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public Locale getLanguageInfo() { - return null; + public Locale getLanguageInfo() throws ParsingException { + return Localization.getLocaleFromThreeLetterCode(data.getString("original_language")); } @Nonnull @Override public List getTags() { - return Arrays.asList(data.getArray("tags").toArray(new String[0])); + return JsonUtils.getStringListFromJsonArray(data.getArray("tags")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 56dbf08fe..3e1ebae79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -286,11 +286,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - try { - return (List) JsonUtils.getArray(json, "tags"); - } catch (Exception e) { - return Collections.emptyList(); - } + return JsonUtils.getStringListFromJsonArray(json.getArray("tags")); } @Nonnull @@ -428,8 +424,19 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getPrivacy() throws ParsingException { - return JsonUtils.getString(json, "privacy.label"); + public Privacy getPrivacy() { + switch (json.getObject("privacy").getInt("id")) { + case 1: + return Privacy.PUBLIC; + case 2: + return Privacy.UNLISTED; + case 3: + return Privacy.PRIVATE; + case 4: + return Privacy.INTERNAL; + default: + return null; + } } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 613cce2cc..1b53b7017 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -374,22 +374,21 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return ""; } - @Nonnull @Override - public String getPrivacy() { - return ""; + public Privacy getPrivacy() { + return track.getString("sharing").equals("public") ? Privacy.PUBLIC : Privacy.PRIVATE; } @Nonnull @Override public String getCategory() { - return ""; + return track.getString("genre"); } @Nonnull @Override public String getLicence() { - return ""; + return track.getString("license"); } @Override @@ -400,7 +399,29 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - return Collections.emptyList(); + // tags are separated by spaces, but they can be multiple words escaped by quotes " + final String[] tag_list = track.getString("tag_list").split(" "); + final List tags = new ArrayList<>(); + String escapedTag = ""; + boolean isEscaped = false; + for (int i = 0; i < tag_list.length; i++) { + String tag = tag_list[i]; + if (tag.startsWith("\"")) { + escapedTag += tag_list[i].replace("\"", ""); + isEscaped = true; + } else if (isEscaped) { + if (tag.endsWith("\"")) { + escapedTag += " " + tag.replace("\"", ""); + isEscaped = false; + tags.add(escapedTag); + } else { + escapedTag += " " + tag; + } + } else if (!tag.isEmpty()){ + tags.add(tag); + } + } + return tags; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index fcc658372..81522ea7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -34,6 +34,7 @@ import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -214,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // description with more info on links try { String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); - if (description != null && !description.isEmpty()) return new Description(description, Description.HTML); + if (!isNullOrEmpty(description)) return new Description(description, Description.HTML); } catch (final ParsingException ignored) { // age-restricted videos cause a ParsingException here } @@ -1107,20 +1108,32 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getPrivacy() { - return ""; + public Privacy getPrivacy() { + boolean isUnlisted = playerResponse + .getObject("microformat") + .getObject("playerMicroformatRenderer") + .getBoolean("isUnlisted"); + return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC; } @Nonnull @Override public String getCategory() { - return ""; + return playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer") + .getString("category"); } @Nonnull @Override - public String getLicence() { - return ""; + public String getLicence() throws ParsingException { + final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() + .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows") + .getObject(0).getObject("metadataRowRenderer"); + + final JsonArray contents = metadataRowRenderer.getArray("contents"); + final String license = getTextFromObject(contents.getObject(0)); + return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) ? license : "YouTube licence"; } @Override @@ -1131,7 +1144,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - return Collections.emptyList(); + return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 3596f2f7c..40eeaf58d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -428,8 +428,7 @@ public abstract class StreamExtractor extends Extractor { * @return the privacy of the stream or an empty String. * @throws ParsingException */ - @Nonnull - public abstract String getPrivacy() throws ParsingException; + public abstract Privacy getPrivacy() throws ParsingException; /** * The name of the category of the stream. @@ -467,7 +466,7 @@ public abstract class StreamExtractor extends Extractor { * The list of tags of the stream. * If the tag list is not available you can simply return an empty list. * - * @return the list of tags of the stream or an empty list. + * @return the list of tags of the stream or Collections.emptyList(). * @throws ParsingException */ @Nonnull @@ -510,4 +509,10 @@ public abstract class StreamExtractor extends Extractor { */ @Nonnull public abstract List getMetaInfo() throws ParsingException; + public enum Privacy { + PUBLIC, + UNLISTED, + PRIVATE, + INTERNAL + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 6477c9e4f..7ff8ad9ee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -377,7 +376,7 @@ public class StreamInfo extends Info { private List subtitles = new ArrayList<>(); private String host = ""; - private String privacy = ""; + private StreamExtractor.Privacy privacy; private String category = ""; private String licence = ""; private String support = ""; @@ -635,11 +634,11 @@ public class StreamInfo extends Info { this.host = str; } - public String getPrivacy() { + public StreamExtractor.Privacy getPrivacy() { return this.privacy; } - public void setPrivacy(String str) { + public void setPrivacy(StreamExtractor.Privacy str) { this.privacy = str; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e9eed4126..22d992f54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -151,4 +151,14 @@ public class JsonUtils { final String json = document.getElementsByAttribute(variable).attr(variable); return JsonParser.object().from(json); } + + public static List getStringListFromJsonArray(@Nonnull final JsonArray array) { + final List stringList = new ArrayList<>(array.size()); + for (Object o : array) { + if (o instanceof String) { + stringList.add((String) o); + } + } + return stringList; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index 69e10b0ed..fd528e7ee 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -9,6 +10,7 @@ import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -69,7 +71,8 @@ public class ExtractorAsserts { } // this assumes that sorting a and b in-place is not an issue, so it's only intended for tests - public static void assertEqualsOrderIndependent(List expected, List actual) { + public static void assertEqualsOrderIndependent(final List expected, + final List actual) { if (expected == null) { assertNull(actual); return; @@ -79,6 +82,7 @@ public class ExtractorAsserts { Collections.sort(expected); Collections.sort(actual); - assertEquals(expected, actual); + // using new ArrayList<> to make sure the type is the same + assertEquals(new ArrayList<>(expected), new ArrayList<>(actual)); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index a42b063b6..09bc4c72d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -66,7 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); } @Override public long expectedLength() { return 3097; } @Override public long expectedViewCountAtLeast() { return 2380; } - @Nullable @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; } - @Override public long expectedLikeCountAtLeast() { return -1; } - @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public boolean expectedHasRelatedStreams() { return false; } - @Override public boolean expectedHasSubtitles() { return false; } - @Override public boolean expectedHasFrames() { return false; } - @Override public List expectedTags() { return Arrays.asList("gpn18", "105"); } - @Override public int expectedStreamSegmentsCount() { return 0; } + @Nullable + @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; } + @Nullable + @Override + public String expectedTextualUploadDate() { + return "2018-05-11T02:00:00.000+02:00"; + } + + @Override + public long expectedLikeCountAtLeast() { + return -1; + } + + @Override + public long expectedDislikeCountAtLeast() { + return -1; + } + + @Override + public boolean expectedHasRelatedStreams() { + return false; + } + + @Override + public boolean expectedHasSubtitles() { + return false; + } + + @Override + public boolean expectedHasFrames() { + return false; + } + + @Override + public List expectedTags() { + return Arrays.asList("gpn18", "105"); + } + + @Override + public int expectedStreamSegmentsCount() { + return 0; + } @Override @Test @@ -86,6 +120,11 @@ public class MediaCCCStreamExtractorTest { super.testAudioStreams(); assertEquals(2, extractor.getAudioStreams().size()); } + + @Override + public Locale expectedLanguageInfo() { + return new Locale("de"); + } } public static class _36c3PrivacyMessaging extends DefaultStreamExtractorTest { @@ -100,27 +139,107 @@ public class MediaCCCStreamExtractorTest { extractor.fetchPage(); } - @Override public StreamExtractor extractor() { return extractor; } - @Override public StreamingService expectedService() { return MediaCCC; } - @Override public String expectedName() { return "What's left for private messaging?"; } - @Override public String expectedId() { return ID; } - @Override public String expectedUrlContains() { return URL; } - @Override public String expectedOriginalUrlContains() { return URL; } + @Override + public StreamExtractor extractor() { + return extractor; + } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } - @Override public String expectedUploaderName() { return "36c3"; } - @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; } - @Override public List expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); } - @Override public long expectedLength() { return 3603; } - @Override public long expectedViewCountAtLeast() { return 2380; } - @Nullable @Override public String expectedUploadDate() { return "2020-01-11 00:00:00.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; } - @Override public long expectedLikeCountAtLeast() { return -1; } - @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public boolean expectedHasRelatedStreams() { return false; } - @Override public boolean expectedHasSubtitles() { return false; } - @Override public boolean expectedHasFrames() { return false; } - @Override public List expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); } + @Override + public StreamingService expectedService() { + return MediaCCC; + } + + @Override + public String expectedName() { + return "What's left for private messaging?"; + } + + @Override + public String expectedId() { + return ID; + } + + @Override + public String expectedUrlContains() { + return URL; + } + + @Override + public String expectedOriginalUrlContains() { + return URL; + } + + @Override + public StreamType expectedStreamType() { + return StreamType.VIDEO_STREAM; + } + + @Override + public String expectedUploaderName() { + return "36c3"; + } + + @Override + public String expectedUploaderUrl() { + return "https://media.ccc.de/c/36c3"; + } + + @Override + public List expectedDescriptionContains() { + return Arrays.asList("WhatsApp", "Signal"); + } + + @Override + public long expectedLength() { + return 3603; + } + + @Override + public long expectedViewCountAtLeast() { + return 2380; + } + + @Nullable + @Override + public String expectedUploadDate() { + return "2020-01-11 00:00:00.000"; + } + + @Nullable + @Override + public String expectedTextualUploadDate() { + return "2020-01-11T01:00:00.000+01:00"; + } + + @Override + public long expectedLikeCountAtLeast() { + return -1; + } + + @Override + public long expectedDislikeCountAtLeast() { + return -1; + } + + @Override + public boolean expectedHasRelatedStreams() { + return false; + } + + @Override + public boolean expectedHasSubtitles() { + return false; + } + + @Override + public boolean expectedHasFrames() { + return false; + } + + @Override + public List expectedTags() { + return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); + } @Override @Test @@ -149,5 +268,10 @@ public class MediaCCCStreamExtractorTest { super.testAudioStreams(); assertEquals(2, extractor.getAudioStreams().size()); } + + @Override + public Locale expectedLanguageInfo() { + return new Locale("en"); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index 0b645508a..f6d040118 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -88,7 +88,6 @@ public class PeertubeStreamExtractorTest { @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public String expectedHost() { return "framatube.org"; } - @Override public String expectedPrivacy() { return "Public"; } @Override public String expectedCategory() { return "Science & Technology"; } @Override public String expectedLicence() { return "Attribution - Share Alike"; } @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); } @@ -139,7 +138,6 @@ public class PeertubeStreamExtractorTest { @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public String expectedHost() { return "nocensoring.net"; } - @Override public String expectedPrivacy() { return "Public"; } @Override public String expectedCategory() { return "Art"; } @Override public String expectedLicence() { return "Attribution"; } @Override public List expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index ddf95c31e..beef559ed 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -34,11 +34,15 @@ public class SoundcloudStreamExtractorTest { private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; private static StreamExtractor extractor; - @Test(expected = GeographicRestrictionException.class) - public void geoRestrictedContent() throws Exception { + @BeforeClass + public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); - extractor.fetchPage(); + try { + extractor.fetchPage(); + } catch (final GeographicRestrictionException e) { + // expected + } } @Override public StreamExtractor extractor() { return extractor; } @@ -67,6 +71,8 @@ public class SoundcloudStreamExtractorTest { @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } @Override public boolean expectedHasRelatedStreams() { return false; } + @Override public String expectedLicence() { return "all-rights-reserved"; } + @Override public String expectedCategory() { return "Pop"; } } public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { @@ -76,11 +82,15 @@ public class SoundcloudStreamExtractorTest { private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; private static StreamExtractor extractor; - @Test(expected = SoundCloudGoPlusContentException.class) - public void goPlusContent() throws Exception { + @BeforeClass + public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); - extractor.fetchPage(); + try { + extractor.fetchPage(); + } catch (final SoundCloudGoPlusContentException e) { + // expected + } } @Override public StreamExtractor extractor() { return extractor; } @@ -109,6 +119,8 @@ public class SoundcloudStreamExtractorTest { @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } + @Override public String expectedLicence() { return "all-rights-reserved"; } + @Override public String expectedCategory() { return "Dance"; } } public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest { @@ -148,6 +160,11 @@ public class SoundcloudStreamExtractorTest { @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } + @Override public String expectedLicence() { return "cc-by"; } + @Override public String expectedCategory() { return "Podcast"; } + @Override public List expectedTags() { + return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia"); + } @Override @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 63fa34119..b52778195 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -54,4 +54,16 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac @Override public int expectedAgeLimit() { return 18; } @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; } @Override public boolean expectedHasSubtitles() { return false; } + + @Override public String expectedCategory() {return "Entertainment"; } + @Override public String expectedLicence() { return "YouTube licence"; } + @Override + public List expectedTags() { + return Arrays.asList("AEE", "AEE 2017", "AVN", "AVN 2016", "AVN 2017", "AVN 2017 Expo In Las Vegas", + "AVN Awards Show", "AVN Expo", "AVN Las Vegas", "AVN Magazine", "AVN Vlog", "Ariana Marie", + "August Ames", "Brenna Sparks", "CeCe Capella", "Cindy Starfall", "Elsa Jean", "Emma Hix", + "FINGERING", "FINGERING P0RNSTARS", "FINGERING PORNSTARS", "Kaho Shibuya", "Keisha Grey", + "Kimberly Chi", "Las Vegas", "Mia Martinez", "Pornstar", "Pornstars", "Riley Reid", + "Samantha Saint", "Vegas", "Vicki Chase"); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 7763e3b3e..0ec370be6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -56,4 +56,8 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; } @Override public long expectedLikeCountAtLeast() { return 13300; } @Override public long expectedDislikeCountAtLeast() { return 2600; } + @Override public List expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); } + @Override public String expectedCategory() { return "Entertainment"; } + @Override public String expectedLicence() { return "YouTube licence"; } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 7990d09ac..c30a282df 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -4,6 +4,7 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.downloader.DownloaderFactory; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -15,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.PrivateContentException; import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamSegment; @@ -56,6 +58,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; public class YoutubeStreamExtractorDefaultTest { private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/"; static final String BASE_URL = "https://www.youtube.com/watch?v="; + public static final String YOUTUBE_LICENCE = "YouTube licence"; public static class NotAvailable { @BeforeClass @@ -145,6 +148,8 @@ public class YoutubeStreamExtractorDefaultTest { @Override public long expectedLikeCountAtLeast() { return 5212900; } @Override public long expectedDislikeCountAtLeast() { return 30600; } @Override public int expectedStreamSegmentsCount() { return 0; } + @Override public String expectedLicence() { return YOUTUBE_LICENCE; } + @Override public String expectedCategory() { return "Entertainment"; } // @formatter:on } @@ -185,6 +190,16 @@ public class YoutubeStreamExtractorDefaultTest { @Override public long expectedLikeCountAtLeast() { return 340100; } @Override public long expectedDislikeCountAtLeast() { return 18700; } @Override public boolean expectedUploaderVerified() { return true; } + @Override public String expectedLicence() { return YOUTUBE_LICENCE; } + @Override public String expectedCategory() { return "Science & Technology"; } + @Override public List expectedTags() { + return Arrays.asList("2018", "8 plus", "apple", "apple iphone", "apple iphone x", "best", "best android", + "best smartphone", "cool gadgets", "find", "find x", "find x review", "find x unboxing", "findx", + "galaxy s9", "galaxy s9+", "hands on", "iphone 8", "iphone 8 plus", "iphone x", "new iphone", "nex", + "oneplus 6", "oppo", "oppo find x", "oppo find x hands on", "oppo find x review", + "oppo find x unboxing", "oppo findx", "pixel 2 xl", "review", "samsung", "samsung galaxy", + "samsung galaxy s9", "smartphone", "unbox therapy", "unboxing", "vivo", "vivo apex", "vivo nex"); + } // @formatter:on } @@ -260,8 +275,16 @@ public class YoutubeStreamExtractorDefaultTest { @Override public long expectedLikeCountAtLeast() { return 32100; } @Override public long expectedDislikeCountAtLeast() { return 750; } @Override public boolean expectedHasSubtitles() { return false; } - @Override public int expectedStreamSegmentsCount() { return 17; } + @Override public String expectedLicence() { return YOUTUBE_LICENCE; } + @Override public String expectedCategory() { return "Music"; } + @Override public List expectedTags() { + return Arrays.asList("2019", "2019 anime", "Anime OST", "Epic anime ost", "OST Anime", + "anime epic soundtrack", "armin", "attack on titan", "battle anime ost", "battle anime soundtracks", + "combat anime ost", "epic soundtrack", "eren", "mikasa", "motivational anime ost", + "motivational anime soundtracks", "shingeki no kyojin"); + } + // @formatter:on @Test public void testStreamSegment() throws Exception { final StreamSegment segment = extractor.getStreamSegments().get(3); @@ -270,7 +293,6 @@ public class YoutubeStreamExtractorDefaultTest { assertEquals(BASE_URL + ID + "?t=589", segment.getUrl()); assertNotNull(segment.getPreviewUrl()); } - // @formatter:on } public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest { @@ -308,6 +330,16 @@ public class YoutubeStreamExtractorDefaultTest { @Override public long expectedDislikeCountAtLeast() { return 20000; } @Override public boolean expectedHasSubtitles() { return true; } @Override public int expectedStreamSegmentsCount() { return 7; } + @Override public String expectedLicence() { return YOUTUBE_LICENCE; } + @Override public String expectedCategory() { return "Science & Technology"; } + @Override public List expectedTags() { + return Arrays.asList("Diabetes", "Erkältung", "Gesundheit", "Immunabwehr", "Immunsystem", "Infektion", + "Komisch alles chemisch", "Krebs", "Lab", "Lesch", "Mai", "Mai Thi", "Mai Thi Nguyen-Kim", + "Mangel", "Nahrungsergänzungsmittel", "Nguyen", "Nguyen Kim", "Nguyen-Kim", "Quarks", "Sommer", + "Supplemente", "Supplements", "Tabletten", "Terra X", "TerraX", "The Secret Life Of Scientists", + "Tropfen", "Vitamin D", "Vitamin-D-Mangel", "Vitamine", "Winter", "einnehmen", "maiLab", "nehmen", + "supplementieren", "Überdosis", "Überschuss"); + } // @formatter:on @Test @@ -369,7 +401,54 @@ public class YoutubeStreamExtractorDefaultTest { )); } @Override public boolean expectedUploaderVerified() { return true; } + @Override public String expectedLicence() { return YOUTUBE_LICENCE; } + @Override public String expectedCategory() { return "Education"; } + @Override public List expectedTags() { + return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus", + "Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf", + "Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial", + "Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale", + "Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische", + "Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale"); + } // @formatter:on } + public static class UnlistedTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM"); + extractor.fetchPage(); + } + + @Test + public void testGetUnlisted() { + assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy()); + } + } + + public static class CCLicensed { + // StreamSegment example with macro-makers panel and transcription panel + private static final String ID = "M4gD1WSo5mA"; + private static final String URL = BASE_URL + ID; + private static StreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getStreamExtractor(URL); + extractor.fetchPage(); + } + + @Test + public void testGetLicence() throws ParsingException { + assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence()); + } + + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 47bf775a9..6e26a7cdb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -39,13 +39,13 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor @Override public String expectedOriginalUrlContains() { return URL; } @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; } - @Override public String expectedUploaderName() { return "ChilledCow"; } + @Override public String expectedUploaderName() { return "Lofi Girl"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; } @Override public List expectedDescriptionContains() { return Arrays.asList("https://bit.ly/chilledcow-playlists", "https://bit.ly/chilledcow-submissions"); } - @Override public boolean expectedUploaderVerified() { return true; } + @Override public boolean expectedUploaderVerified() { return false; } @Override public long expectedLength() { return 0; } @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 0; } @@ -56,4 +56,14 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor @Override public boolean expectedHasSubtitles() { return false; } @Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; } @Override public boolean expectedHasFrames() { return false; } + @Override public String expectedLicence() { return "YouTube licence"; } + @Override public String expectedCategory() { return "Music"; } + @Override public List expectedTags() { + return Arrays.asList("beats to relax", "chilled cow", "chilled cow radio", "chilledcow", "chilledcow radio", + "chilledcow station", "chillhop", "hip hop", "hiphop", "lo fi", "lo fi hip hop", "lo fi hip hop radio", + "lo fi hiphop", "lo fi radio", "lo-fi", "lo-fi hip hop", "lo-fi hip hop radio", "lo-fi hiphop", + "lo-fi radio", "lofi", "lofi hip hop", "lofi hip hop radio", "lofi hiphop", "lofi radio", "music", + "lofi radio chilledcow", "music to study", "playlist", "radio", "relaxing music", "study music", + "lofi hip hop radio - beats to relax\\/study to"); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java index 83bd8ae7f..4ef0e7121 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -15,6 +15,7 @@ import java.util.List; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED; public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest { private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/"; @@ -50,4 +51,8 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe @Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; } @Override public long expectedLikeCountAtLeast() { return 110; } @Override public long expectedDislikeCountAtLeast() { return 0; } + @Override public StreamExtractor.Privacy expectedPrivacy() { return UNLISTED; } + @Override public String expectedLicence() { return "YouTube licence"; } + @Override public String expectedCategory() { return "Gaming"; } + @Override public List expectedTags() { return Arrays.asList("dark souls", "hooked", "praise the casual"); } }