Merge pull request #306 from B0pol/metadata

Extract metadata for youtube, soundcloud & mediaccc

extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor.localization;
 
-import edu.umd.cs.findbugs.annotations.NonNull;
 
+import javax.annotation.Nonnull;
 import java.io.Serializable;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -12,14 +12,15 @@ import java.util.GregorianCalendar;
  * A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
  */
 public class DateWrapper implements Serializable {
-    @NonNull private final OffsetDateTime offsetDateTime;
+    @Nonnull
+    private final OffsetDateTime offsetDateTime;
     private final boolean isApproximation;
 
     /**
      * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
      */
     @Deprecated
-    public DateWrapper(@NonNull Calendar calendar) {
+    public DateWrapper(@Nonnull Calendar calendar) {
         this(calendar, false);
     }
 
@@ -27,26 +28,25 @@ public class DateWrapper implements Serializable {
      * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
      */
     @Deprecated
-    public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
+    public DateWrapper(@Nonnull Calendar calendar, boolean isApproximation) {
         this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation);
     }
 
-    public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
+    public DateWrapper(@Nonnull OffsetDateTime offsetDateTime) {
         this(offsetDateTime, false);
     }
 
-    public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
+    public DateWrapper(@Nonnull OffsetDateTime offsetDateTime, boolean isApproximation) {
         this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
         this.isApproximation = isApproximation;
     }
 
     /**
      * @return the wrapped date/time as a {@link Calendar}.
-     *
      * @deprecated use {@link #offsetDateTime()} instead.
      */
     @Deprecated
-    @NonNull
+    @Nonnull
     public Calendar date() {
         return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
     }
@@ -54,7 +54,7 @@ public class DateWrapper implements Serializable {
     /**
      * @return the wrapped date/time.
      */
-    @NonNull
+    @Nonnull
     public OffsetDateTime offsetDateTime() {
         return offsetDateTime;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java

@@ -1,19 +1,19 @@
 package org.schabi.newpipe.extractor.localization;
 
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
 
 public class Localization implements Serializable {
     public static final Localization DEFAULT = new Localization("en", "GB");
 
-    @Nonnull private final String languageCode;
+    @Nonnull
-    @Nullable private final String countryCode;
+    private final String languageCode;
+    @Nullable
+    private final String countryCode;
 
     /**
      * @param localizationCodeList a list of localization code, formatted like {@link #getLocalizationCode()}
@@ -100,4 +100,25 @@ public class Localization implements Serializable {
         result = 31 * result + Objects.hashCode(countryCode);
         return result;
     }
+
+    /**
+     * Converts a three letter language code (ISO 639-2/T) to a Locale
+     * because limits of Java Locale class.
+     *
+     * @param code a three letter language code
+     * @return the Locale corresponding
+     */
+    public static Locale getLocaleFromThreeLetterCode(@Nonnull String code) throws ParsingException {
+        final String[] languages = Locale.getISOLanguages();
+        final Map<String, Locale> localeMap = new HashMap<>(languages.length);
+        for (String language : languages) {
+            final Locale locale = new Locale(language);
+            localeMap.put(locale.getISO3Language(), locale);
+        }
+        if (localeMap.containsKey(code)) {
+            return localeMap.get(code);
+        } else {
+            throw new ParsingException("Could not get Locale from this three letter language code" + code);
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java

@@ -140,4 +140,10 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
     public List<String> getTags() {
         return Collections.emptyList();
     }
+
+    @Nonnull
+    @Override
+    public Privacy getPrivacy() {
+        return Privacy.PUBLIC;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java

@@ -262,8 +262,8 @@ public class BandcampStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getPrivacy() {
+    public Privacy getPrivacy() {
-        return "";
+        return Privacy.PUBLIC;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -257,8 +257,8 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getPrivacy() {
+    public Privacy getPrivacy() {
-        return "Public";
+        return Privacy.PUBLIC;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -12,14 +12,19 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
 
 public class MediaCCCStreamExtractor extends StreamExtractor {
     private JsonObject data;
@@ -256,8 +261,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getPrivacy() {
+    public Privacy getPrivacy() {
-        return "";
+        return Privacy.PUBLIC;
     }
 
     @Nonnull
@@ -273,14 +278,14 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public Locale getLanguageInfo() {
+    public Locale getLanguageInfo() throws ParsingException {
-        return null;
+        return Localization.getLocaleFromThreeLetterCode(data.getString("original_language"));
     }
 
     @Nonnull
     @Override
     public List<String> getTags() {
-        return Arrays.asList(data.getArray("tags").toArray(new String[0]));
+        return JsonUtils.getStringListFromJsonArray(data.getArray("tags"));
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -286,11 +286,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        try {
+        return JsonUtils.getStringListFromJsonArray(json.getArray("tags"));
-            return (List) JsonUtils.getArray(json, "tags");
-        } catch (Exception e) {
-            return Collections.emptyList();
-        }
     }
 
     @Nonnull
@@ -428,8 +424,19 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getPrivacy() throws ParsingException {
+    public Privacy getPrivacy() {
-        return JsonUtils.getString(json, "privacy.label");
+        switch (json.getObject("privacy").getInt("id")) {
+            case 1:
+                return Privacy.PUBLIC;
+            case 2:
+                return Privacy.UNLISTED;
+            case 3:
+                return Privacy.PRIVATE;
+            case 4:
+                return Privacy.INTERNAL;
+            default:
+                return Privacy.OTHER;
+        }
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -374,22 +374,21 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return "";
     }
 
-    @Nonnull
     @Override
-    public String getPrivacy() {
+    public Privacy getPrivacy() {
-        return "";
+        return track.getString("sharing").equals("public") ? Privacy.PUBLIC : Privacy.PRIVATE;
     }
 
     @Nonnull
     @Override
     public String getCategory() {
-        return "";
+        return track.getString("genre");
     }
 
     @Nonnull
     @Override
     public String getLicence() {
-        return "";
+        return track.getString("license");
     }
 
     @Override
@@ -400,7 +399,29 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        return Collections.emptyList();
+        // tags are separated by spaces, but they can be multiple words  escaped by quotes "
+        final String[] tag_list = track.getString("tag_list").split(" ");
+        final List<String> tags = new ArrayList<>();
+        String escapedTag = "";
+        boolean isEscaped = false;
+        for (int i = 0; i < tag_list.length; i++) {
+            String tag = tag_list[i];
+            if (tag.startsWith("\"")) {
+                escapedTag += tag_list[i].replace("\"", "");
+                isEscaped = true;
+            } else if (isEscaped) {
+                if (tag.endsWith("\"")) {
+                    escapedTag += " " + tag.replace("\"", "");
+                    isEscaped = false;
+                    tags.add(escapedTag);
+                } else {
+                    escapedTag += " " + tag;
+                }
+            } else if (!tag.isEmpty()){
+                tags.add(tag);
+            }
+        }
+        return tags;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -34,6 +34,7 @@ import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -214,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // description with more info on links
         try {
             String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
-            if (description != null && !description.isEmpty()) return new Description(description, Description.HTML);
+            if (!isNullOrEmpty(description)) return new Description(description, Description.HTML);
         } catch (final ParsingException ignored) {
             // age-restricted videos cause a ParsingException here
         }
@@ -1107,20 +1108,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getPrivacy() {
+    public Privacy getPrivacy() {
-        return "";
+        final boolean isUnlisted = playerResponse
+                .getObject("microformat")
+                .getObject("playerMicroformatRenderer")
+                .getBoolean("isUnlisted");
+        return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
     }
 
     @Nonnull
     @Override
     public String getCategory() {
-        return "";
+        return playerResponse.getObject("microformat")
+                .getObject("playerMicroformatRenderer")
+                .getString("category");
     }
 
     @Nonnull
     @Override
-    public String getLicence() {
+    public String getLicence() throws ParsingException {
-        return "";
+        final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
+                .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows")
+                .getObject(0).getObject("metadataRowRenderer");
+
+        final JsonArray contents = metadataRowRenderer.getArray("contents");
+        final String license = getTextFromObject(contents.getObject(0));
+        return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) ? license : "YouTube licence";
     }
 
     @Override
@@ -1131,7 +1144,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        return Collections.emptyList();
+        return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords"));
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -428,8 +428,7 @@ public abstract class StreamExtractor extends Extractor {
      * @return the privacy of the stream or an empty String.
      * @throws ParsingException
      */
-    @Nonnull
+    public abstract Privacy getPrivacy() throws ParsingException;
-    public abstract String getPrivacy() throws ParsingException;
 
     /**
      * The name of the category of the stream.
@@ -467,7 +466,7 @@ public abstract class StreamExtractor extends Extractor {
      * The list of tags of the stream.
      * If the tag list is not available you can simply return an empty list.
      *
-     * @return the list of tags of the stream or an empty list.
+     * @return the list of tags of the stream or Collections.emptyList().
      * @throws ParsingException
      */
     @Nonnull
@@ -510,4 +509,11 @@ public abstract class StreamExtractor extends Extractor {
      */
     @Nonnull
     public abstract List<MetaInfo> getMetaInfo() throws ParsingException;
+    public enum Privacy {
+        PUBLIC,
+        UNLISTED,
+        PRIVATE,
+        INTERNAL,
+        OTHER
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
@@ -377,7 +376,7 @@ public class StreamInfo extends Info {
     private List<SubtitlesStream> subtitles = new ArrayList<>();
 
     private String host = "";
-    private String privacy = "";
+    private StreamExtractor.Privacy privacy;
     private String category = "";
     private String licence = "";
     private String support = "";
@@ -635,11 +634,11 @@ public class StreamInfo extends Info {
         this.host = str;
     }
 
-    public String getPrivacy() {
+    public StreamExtractor.Privacy getPrivacy() {
         return this.privacy;
     }
 
-    public void setPrivacy(String str) {
+    public void setPrivacy(StreamExtractor.Privacy str) {
         this.privacy = str;
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -151,4 +151,14 @@ public class JsonUtils {
         final String json = document.getElementsByAttribute(variable).attr(variable);
         return JsonParser.object().from(json);
     }
+
+    public static List<String> getStringListFromJsonArray(@Nonnull final JsonArray array) {
+        final List<String> stringList = new ArrayList<>(array.size());
+        for (Object o : array) {
+            if (o instanceof String) {
+                stringList.add((String) o);
+            }
+        }
+        return stringList;
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
 
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -9,6 +10,7 @@ import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -69,7 +71,8 @@ public class ExtractorAsserts {
     }
 
     // this assumes that sorting a and b in-place is not an issue, so it's only intended for tests
-    public static void assertEqualsOrderIndependent(List<String> expected, List<String> actual) {
+    public static void assertEqualsOrderIndependent(final List<String> expected,
+                                                    final List<String> actual) {
         if (expected == null) {
             assertNull(actual);
             return;
@@ -79,6 +82,7 @@ public class ExtractorAsserts {
 
         Collections.sort(expected);
         Collections.sort(actual);
-        assertEquals(expected, actual);
+        // using new ArrayList<> to make sure the type is the same
+        assertEquals(new ArrayList<>(expected), new ArrayList<>(actual));
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java

@@ -66,7 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
     @Nullable public String expectedDashMpdUrlContains() { return null; } // default: no dash mpd
     public boolean expectedHasFrames() { return true; } // default: there are frames
     public String expectedHost() { return ""; } // default: no host for centralized platforms
-    public String expectedPrivacy() { return ""; } // default: no privacy policy available
+    public StreamExtractor.Privacy expectedPrivacy() { return StreamExtractor.Privacy.PUBLIC; } // default: public
     public String expectedCategory() { return ""; } // default: no category
     public String expectedLicence() { return ""; } // default: no licence
     public Locale expectedLanguageInfo() { return null; } // default: no language info available

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java

@@ -5,15 +5,17 @@ import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
+import javax.annotation.Nullable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-
+import java.util.Locale;
-import javax.annotation.Nullable;
 
 import static junit.framework.TestCase.assertEquals;
 import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@@ -42,7 +44,6 @@ public class MediaCCCStreamExtractorTest {
         @Override public String expectedId() { return ID; }
         @Override public String expectedUrlContains() { return URL; }
         @Override public String expectedOriginalUrlContains() { return URL; }
-
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "gpn18"; }
         @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; }
@@ -58,10 +59,10 @@ public class MediaCCCStreamExtractorTest {
         @Override public boolean expectedHasFrames() { return false; }
         @Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
         @Override public int expectedStreamSegmentsCount() { return 0; }
+        @Override public Locale expectedLanguageInfo() { return new Locale("de"); }
 
         @Override
-        @Test
+        @Test public void testThumbnailUrl() throws Exception {
-        public void testThumbnailUrl() throws Exception {
             super.testThumbnailUrl();
             assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
         }
@@ -100,13 +101,20 @@ public class MediaCCCStreamExtractorTest {
             extractor.fetchPage();
         }
 
-        @Override public StreamExtractor extractor() { return extractor; }
+        @Override public StreamExtractor extractor() {
-        @Override public StreamingService expectedService() { return MediaCCC; }
+            return extractor;
-        @Override public String expectedName() { return "What's left for private messaging?"; }
+        }
-        @Override public String expectedId() { return ID; }
+        @Override public StreamingService expectedService() {
+            return MediaCCC;
+        }
+        @Override public String expectedName() {
+            return "What's left for private messaging?";
+        }
+        @Override public String expectedId() {
+            return ID;
+        }
         @Override public String expectedUrlContains() { return URL; }
         @Override public String expectedOriginalUrlContains() { return URL; }
-
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "36c3"; }
         @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; }
@@ -123,8 +131,7 @@ public class MediaCCCStreamExtractorTest {
         @Override public List<String> expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); }
 
         @Override
-        @Test
+        @Test public void testThumbnailUrl() throws Exception {
-        public void testThumbnailUrl() throws Exception {
             super.testThumbnailUrl();
             assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
         }
@@ -149,5 +156,10 @@ public class MediaCCCStreamExtractorTest {
             super.testAudioStreams();
             assertEquals(2, extractor.getAudioStreams().size());
         }
+
+        @Override
+        public Locale expectedLanguageInfo() {
+            return new Locale("en");
+        }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -88,7 +88,6 @@ public class PeertubeStreamExtractorTest {
         @Override public boolean expectedHasAudioStreams() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public String expectedHost() { return "framatube.org"; }
-        @Override public String expectedPrivacy() { return "Public"; }
         @Override public String expectedCategory() { return "Science & Technology"; }
         @Override public String expectedLicence() { return "Attribution - Share Alike"; }
         @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@@ -139,7 +138,6 @@ public class PeertubeStreamExtractorTest {
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public String expectedHost() { return "nocensoring.net"; }
-        @Override public String expectedPrivacy() { return "Public"; }
         @Override public String expectedCategory() { return "Art"; }
         @Override public String expectedLicence() { return "Attribution"; }
         @Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -34,11 +34,15 @@ public class SoundcloudStreamExtractorTest {
         private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
         private static StreamExtractor extractor;
 
-        @Test(expected = GeographicRestrictionException.class)
+        @BeforeClass
-        public void geoRestrictedContent() throws Exception {
+        public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = SoundCloud.getStreamExtractor(URL);
-            extractor.fetchPage();
+            try {
+                extractor.fetchPage();
+            } catch (final GeographicRestrictionException e) {
+                // expected
+            }
         }
 
         @Override public StreamExtractor extractor() { return extractor; }
@@ -67,6 +71,8 @@ public class SoundcloudStreamExtractorTest {
         @Override public boolean expectedHasFrames() { return false; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
         @Override public boolean expectedHasRelatedStreams() { return false; }
+        @Override public String expectedLicence() { return "all-rights-reserved"; }
+        @Override public String expectedCategory() { return "Pop"; }
     }
 
     public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest {
@@ -76,11 +82,15 @@ public class SoundcloudStreamExtractorTest {
         private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
         private static StreamExtractor extractor;
 
-        @Test(expected = SoundCloudGoPlusContentException.class)
+        @BeforeClass
-        public void goPlusContent() throws Exception {
+        public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = SoundCloud.getStreamExtractor(URL);
-            extractor.fetchPage();
+            try {
+                extractor.fetchPage();
+            } catch (final SoundCloudGoPlusContentException e) {
+                // expected
+            }
         }
 
         @Override public StreamExtractor extractor() { return extractor; }
@@ -109,6 +119,8 @@ public class SoundcloudStreamExtractorTest {
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
+        @Override public String expectedLicence() { return "all-rights-reserved"; }
+        @Override public String expectedCategory() { return "Dance"; }
     }
 
     public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
@@ -148,6 +160,11 @@ public class SoundcloudStreamExtractorTest {
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
+        @Override public String expectedLicence() { return "cc-by"; }
+        @Override public String expectedCategory() { return "Podcast"; }
+        @Override public List<String> expectedTags() {
+            return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia");
+        }
 
         @Override
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -54,4 +54,16 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
     @Override public int expectedAgeLimit() { return 18; }
     @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
     @Override public boolean expectedHasSubtitles() { return false; }
+
+    @Override public String expectedCategory() {return "Entertainment"; }
+    @Override public String expectedLicence() { return "YouTube licence"; }
+    @Override
+    public List<String> expectedTags() {
+        return Arrays.asList("AEE", "AEE 2017", "AVN", "AVN 2016", "AVN 2017", "AVN 2017 Expo In Las Vegas",
+                "AVN Awards Show", "AVN Expo", "AVN Las Vegas", "AVN Magazine", "AVN Vlog", "Ariana Marie",
+                "August Ames", "Brenna Sparks", "CeCe Capella", "Cindy Starfall", "Elsa Jean", "Emma Hix",
+                "FINGERING", "FINGERING P0RNSTARS", "FINGERING PORNSTARS", "Kaho Shibuya", "Keisha Grey",
+                "Kimberly Chi", "Las Vegas", "Mia Martinez", "Pornstar", "Pornstars", "Riley Reid",
+                "Samantha Saint", "Vegas", "Vicki Chase");
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -56,4 +56,8 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
     @Override public long expectedLikeCountAtLeast() { return 13300; }
     @Override public long expectedDislikeCountAtLeast() { return 2600; }
+    @Override public List<String> expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); }
+    @Override public String expectedCategory() { return "Entertainment"; }
+    @Override public String expectedLicence() { return "YouTube licence"; }
+
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -15,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
 import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamSegment;
@@ -56,6 +58,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 public class YoutubeStreamExtractorDefaultTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     static final String BASE_URL = "https://www.youtube.com/watch?v=";
+    public static final String YOUTUBE_LICENCE = "YouTube licence";
 
     public static class NotAvailable {
         @BeforeClass
@@ -145,6 +148,8 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedLikeCountAtLeast() { return 5212900; }
         @Override public long expectedDislikeCountAtLeast() { return 30600; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
+        @Override public String expectedLicence() { return YOUTUBE_LICENCE; }
+        @Override public String expectedCategory() { return "Entertainment"; }
         // @formatter:on
     }
 
@@ -185,6 +190,16 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedLikeCountAtLeast() { return 340100; }
         @Override public long expectedDislikeCountAtLeast() { return 18700; }
         @Override public boolean expectedUploaderVerified() { return true; }
+        @Override public String expectedLicence() { return YOUTUBE_LICENCE; }
+        @Override public String expectedCategory() { return "Science & Technology"; }
+        @Override public List<String> expectedTags() {
+            return Arrays.asList("2018", "8 plus", "apple", "apple iphone", "apple iphone x", "best", "best android",
+                    "best smartphone", "cool gadgets", "find", "find x", "find x review", "find x unboxing", "findx",
+                    "galaxy s9", "galaxy s9+", "hands on", "iphone 8", "iphone 8 plus", "iphone x", "new iphone", "nex",
+                    "oneplus 6", "oppo", "oppo find x", "oppo find x hands on", "oppo find x review",
+                    "oppo find x unboxing", "oppo findx", "pixel 2 xl", "review", "samsung", "samsung galaxy",
+                    "samsung galaxy s9", "smartphone", "unbox therapy", "unboxing", "vivo", "vivo apex", "vivo nex");
+        }
         // @formatter:on
     }
 
@@ -260,8 +275,17 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedLikeCountAtLeast() { return 32100; }
         @Override public long expectedDislikeCountAtLeast() { return 750; }
         @Override public boolean expectedHasSubtitles() { return false; }
-
         @Override public int expectedStreamSegmentsCount() { return 17; }
+        @Override public String expectedLicence() { return YOUTUBE_LICENCE; }
+        @Override public String expectedCategory() { return "Music"; }
+        @Override public List<String> expectedTags() {
+            return Arrays.asList("2019", "2019 anime", "Anime OST", "Epic anime ost", "OST Anime",
+                    "anime epic soundtrack", "armin", "attack on titan", "battle anime ost", "battle anime soundtracks",
+                    "combat anime ost", "epic soundtrack", "eren", "mikasa", "motivational anime ost",
+                    "motivational anime soundtracks", "shingeki no kyojin");
+        }
+        // @formatter:on
+
         @Test
         public void testStreamSegment() throws Exception {
             final StreamSegment segment = extractor.getStreamSegments().get(3);
@@ -270,7 +294,6 @@ public class YoutubeStreamExtractorDefaultTest {
             assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
             assertNotNull(segment.getPreviewUrl());
         }
-        // @formatter:on
     }
 
     public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@@ -308,6 +331,16 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public long expectedDislikeCountAtLeast() { return 20000; }
         @Override public boolean expectedHasSubtitles() { return true; }
         @Override public int expectedStreamSegmentsCount() { return 7; }
+        @Override public String expectedLicence() { return YOUTUBE_LICENCE; }
+        @Override public String expectedCategory() { return "Science & Technology"; }
+        @Override public List<String> expectedTags() {
+            return Arrays.asList("Diabetes", "Erkältung", "Gesundheit", "Immunabwehr", "Immunsystem", "Infektion",
+                    "Komisch alles chemisch", "Krebs", "Lab", "Lesch", "Mai", "Mai Thi", "Mai Thi Nguyen-Kim",
+                    "Mangel", "Nahrungsergänzungsmittel", "Nguyen", "Nguyen Kim", "Nguyen-Kim", "Quarks", "Sommer",
+                    "Supplemente", "Supplements", "Tabletten", "Terra X", "TerraX", "The Secret Life Of Scientists",
+                    "Tropfen", "Vitamin D", "Vitamin-D-Mangel", "Vitamine", "Winter", "einnehmen", "maiLab", "nehmen",
+                    "supplementieren", "Überdosis", "Überschuss");
+        }
         // @formatter:on
 
         @Test
@@ -322,9 +355,12 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override
         @Test
         @Ignore("encoding problem")
-        public void testName() throws Exception {
+        public void testName() {}
-            super.testName();
+
-        }
+        @Override
+        @Test
+        @Ignore("encoding problem")
+        public void testTags() {}
     }
 
     public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
@@ -369,7 +405,51 @@ public class YoutubeStreamExtractorDefaultTest {
             ));
         }
         @Override public boolean expectedUploaderVerified() { return true; }
+        @Override public String expectedLicence() { return YOUTUBE_LICENCE; }
+        @Override public String expectedCategory() { return "Education"; }
+        @Override public List<String> expectedTags() {
+            return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus",
+                    "Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf",
+                    "Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial",
+                    "Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale",
+                    "Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische",
+                    "Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale");
+        }
         // @formatter:on
     }
 
+    public static class UnlistedTest {
+        private static YoutubeStreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeStreamExtractor) YouTube
+                    .getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetUnlisted() {
+            assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy());
+        }
+    }
+
+    public static class CCLicensed {
+        private static final String ID = "M4gD1WSo5mA";
+        private static final String URL = BASE_URL + ID;
+        private static StreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = YouTube.getStreamExtractor(URL);
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetLicence() throws ParsingException {
+            assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence());
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -39,13 +39,13 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
     @Override public String expectedOriginalUrlContains() { return URL; }
 
     @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
-    @Override public String expectedUploaderName() { return "ChilledCow"; }
+    @Override public String expectedUploaderName() { return "Lofi Girl"; }
     @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
     @Override public List<String> expectedDescriptionContains() {
         return Arrays.asList("https://bit.ly/chilledcow-playlists",
                 "https://bit.ly/chilledcow-submissions");
     }
-    @Override public boolean expectedUploaderVerified() { return true; }
+    @Override public boolean expectedUploaderVerified() { return false; }
     @Override public long expectedLength() { return 0; }
     @Override public long expectedTimestamp() { return TIMESTAMP; }
     @Override public long expectedViewCountAtLeast() { return 0; }
@@ -56,4 +56,14 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
     @Override public boolean expectedHasSubtitles() { return false; }
     @Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; }
     @Override public boolean expectedHasFrames() { return false; }
+    @Override public String expectedLicence() { return "YouTube licence"; }
+    @Override public String expectedCategory() { return "Music"; }
+    @Override public List<String> expectedTags() {
+        return Arrays.asList("beats to relax", "chilled cow", "chilled cow radio", "chilledcow", "chilledcow radio",
+                "chilledcow station", "chillhop", "hip hop", "hiphop", "lo fi", "lo fi hip hop", "lo fi hip hop radio",
+                "lo fi hiphop", "lo fi radio", "lo-fi", "lo-fi hip hop", "lo-fi hip hop radio", "lo-fi hiphop",
+                "lo-fi radio", "lofi", "lofi hip hop", "lofi hip hop radio", "lofi hiphop", "lofi radio", "music",
+                "lofi radio chilledcow", "music to study", "playlist", "radio", "relaxing music", "study music",
+                "lofi hip hop radio - beats to relax\\/study to");
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java

@@ -15,6 +15,7 @@ import java.util.List;
 import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED;
 
 public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
@@ -50,4 +51,8 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
     @Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; }
     @Override public long expectedLikeCountAtLeast() { return 110; }
     @Override public long expectedDislikeCountAtLeast() { return 0; }
+    @Override public StreamExtractor.Privacy expectedPrivacy() { return UNLISTED; }
+    @Override public String expectedLicence() { return "YouTube licence"; }
+    @Override public String expectedCategory() { return "Gaming"; }
+    @Override public List<String> expectedTags() { return Arrays.asList("dark souls", "hooked", "praise the casual"); }
 }