be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Extract metadata for YouTube, SoundCloud & MediaCCC

This commit is contained in:
bopol 2020-02-09 11:59:23 +01:00 committed by Stypox
parent f71cfd489c
commit c47cc54908
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
22 changed files with 440 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull;
import javax.annotation.Nonnull;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
@ -12,14 +12,15 @@ import java.util.GregorianCalendar;
* A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
*/
public class DateWrapper implements Serializable {
@NonNull private final OffsetDateTime offsetDateTime;
@Nonnull
private final OffsetDateTime offsetDateTime;
private final boolean isApproximation;
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar) {
public DateWrapper(@Nonnull Calendar calendar) {
this(calendar, false);
}
@ -27,26 +28,25 @@ public class DateWrapper implements Serializable {
* @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
public DateWrapper(@Nonnull Calendar calendar, boolean isApproximation) {
this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
public DateWrapper(@Nonnull OffsetDateTime offsetDateTime) {
this(offsetDateTime, false);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
public DateWrapper(@Nonnull OffsetDateTime offsetDateTime, boolean isApproximation) {
this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
/**
* @return the wrapped date/time as a {@link Calendar}.
*
* @deprecated use {@link #offsetDateTime()} instead.
*/
@Deprecated
@NonNull
@Nonnull
public Calendar date() {
return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
}
@ -54,7 +54,7 @@ public class DateWrapper implements Serializable {
/**
* @return the wrapped date/time.
*/
@NonNull
@Nonnull
public OffsetDateTime offsetDateTime() {
return offsetDateTime;
}

35
extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java
View File

@ -1,19 +1,19 @@
package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.*;
public class Localization implements Serializable {
public static final Localization DEFAULT = new Localization("en", "GB");
@Nonnull private final String languageCode;
@Nullable private final String countryCode;
@Nonnull
private final String languageCode;
@Nullable
private final String countryCode;
/**
* @param localizationCodeList a list of localization code, formatted like {@link #getLocalizationCode()}
@ -100,4 +100,25 @@ public class Localization implements Serializable {
result = 31 * result + Objects.hashCode(countryCode);
return result;
}
/**
* Converts a three letter language code (ISO 639-2/T) to a Locale
* in the limit of Java Locale class.
*
* @param code a three letter language code
* @return the Locale corresponding
*/
public static Locale getLocaleFromThreeLetterCode(@Nonnull String code) throws ParsingException {
String[] languages = Locale.getISOLanguages();
Map<String, Locale> localeMap = new HashMap<>(languages.length);
for (String language : languages) {
final Locale locale = new Locale(language);
localeMap.put(locale.getISO3Language(), locale);
}
if (localeMap.containsKey(code)) {
return localeMap.get(code);
} else {
throw new ParsingException("Could not get Locale from this three letter language code" + code);
}
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java
View File

@ -140,4 +140,10 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
public List<String> getTags() {
return Collections.emptyList();
}
@Nonnull
@Override
public Privacy getPrivacy() {
return Privacy.PUBLIC;
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java
View File

@ -262,8 +262,8 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getPrivacy() {
return "";
public Privacy getPrivacy() {
return Privacy.PUBLIC;
}
@Nonnull

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java
View File

@ -257,8 +257,8 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getPrivacy() {
return "Public";
public Privacy getPrivacy() {
return Privacy.PUBLIC;
}
@Nonnull

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -12,14 +12,19 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
@ -256,8 +261,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getPrivacy() {
return "";
public Privacy getPrivacy() {
return Privacy.PUBLIC;
}
@Nonnull
@ -273,14 +278,14 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
}
@Override
public Locale getLanguageInfo() {
return null;
public Locale getLanguageInfo() throws ParsingException {
return Localization.getLocaleFromThreeLetterCode(data.getString("original_language"));
}
@Nonnull
@Override
public List<String> getTags() {
return Arrays.asList(data.getArray("tags").toArray(new String[0]));
return JsonUtils.getStringListFromJsonArray(data.getArray("tags"));
}
@Nonnull

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -286,11 +286,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
try {
return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) {
return Collections.emptyList();
}
return JsonUtils.getStringListFromJsonArray(json.getArray("tags"));
}
@Nonnull
@ -428,8 +424,19 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
public Privacy getPrivacy() {
switch (json.getObject("privacy").getInt("id")) {
case 1:
return Privacy.PUBLIC;
case 2:
return Privacy.UNLISTED;
case 3:
return Privacy.PRIVATE;
case 4:
return Privacy.INTERNAL;
default:
return null;
}
}
@Nonnull

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -374,22 +374,21 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return "";
}
@Nonnull
@Override
public String getPrivacy() {
return "";
public Privacy getPrivacy() {
return track.getString("sharing").equals("public") ? Privacy.PUBLIC : Privacy.PRIVATE;
}
@Nonnull
@Override
public String getCategory() {
return "";
return track.getString("genre");
}
@Nonnull
@Override
public String getLicence() {
return "";
return track.getString("license");
}
@Override
@ -400,7 +399,29 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
return Collections.emptyList();
// tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tag_list = track.getString("tag_list").split(" ");
final List<String> tags = new ArrayList<>();
String escapedTag = "";
boolean isEscaped = false;
for (int i = 0; i < tag_list.length; i++) {
String tag = tag_list[i];
if (tag.startsWith("\"")) {
escapedTag += tag_list[i].replace("\"", "");
isEscaped = true;
} else if (isEscaped) {
if (tag.endsWith("\"")) {
escapedTag += " " + tag.replace("\"", "");
isEscaped = false;
tags.add(escapedTag);
} else {
escapedTag += " " + tag;
}
} else if (!tag.isEmpty()){
tags.add(tag);
}
}
return tags;
}
@Nonnull

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -34,6 +34,7 @@ import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -214,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// description with more info on links
try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
if (description != null && !description.isEmpty()) return new Description(description, Description.HTML);
if (!isNullOrEmpty(description)) return new Description(description, Description.HTML);
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
@ -1107,20 +1108,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getPrivacy() {
return "";
public Privacy getPrivacy() {
boolean isUnlisted = playerResponse
.getObject("microformat")
.getObject("playerMicroformatRenderer")
.getBoolean("isUnlisted");
return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
}
@Nonnull
@Override
public String getCategory() {
return "";
return playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer")
.getString("category");
}
@Nonnull
@Override
public String getLicence() {
return "";
public String getLicence() throws ParsingException {
final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows")
.getObject(0).getObject("metadataRowRenderer");
final JsonArray contents = metadataRowRenderer.getArray("contents");
final String license = getTextFromObject(contents.getObject(0));
return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) ? license : "YouTube licence";
}
@Override
@ -1131,7 +1144,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
return Collections.emptyList();
return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords"));
}
@Nonnull

11
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -428,8 +428,7 @@ public abstract class StreamExtractor extends Extractor {
* @return the privacy of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getPrivacy() throws ParsingException;
public abstract Privacy getPrivacy() throws ParsingException;
/**
* The name of the category of the stream.
@ -467,7 +466,7 @@ public abstract class StreamExtractor extends Extractor {
* The list of tags of the stream.
* If the tag list is not available you can simply return an empty list.
*
* @return the list of tags of the stream or an empty list.
* @return the list of tags of the stream or Collections.emptyList().
* @throws ParsingException
*/
@Nonnull
@ -510,4 +509,10 @@ public abstract class StreamExtractor extends Extractor {
*/
@Nonnull
public abstract List<MetaInfo> getMetaInfo() throws ParsingException;
public enum Privacy {
PUBLIC,
UNLISTED,
PRIVATE,
INTERNAL
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
@ -377,7 +376,7 @@ public class StreamInfo extends Info {
private List<SubtitlesStream> subtitles = new ArrayList<>();
private String host = "";
private String privacy = "";
private StreamExtractor.Privacy privacy;
private String category = "";
private String licence = "";
private String support = "";
@ -635,11 +634,11 @@ public class StreamInfo extends Info {
this.host = str;
}
public String getPrivacy() {
public StreamExtractor.Privacy getPrivacy() {
return this.privacy;
}
public void setPrivacy(String str) {
public void setPrivacy(StreamExtractor.Privacy str) {
this.privacy = str;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -151,4 +151,14 @@ public class JsonUtils {
final String json = document.getElementsByAttribute(variable).attr(variable);
return JsonParser.object().from(json);
}
public static List<String> getStringListFromJsonArray(@Nonnull final JsonArray array) {
final List<String> stringList = new ArrayList<>(array.size());
for (Object o : array) {
if (o instanceof String) {
stringList.add((String) o);
}
}
return stringList;
}
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@ -9,6 +10,7 @@ import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@ -69,7 +71,8 @@ public class ExtractorAsserts {
}
// this assumes that sorting a and b in-place is not an issue, so it's only intended for tests
public static void assertEqualsOrderIndependent(List<String> expected, List<String> actual) {
public static void assertEqualsOrderIndependent(final List<String> expected,
final List<String> actual) {
if (expected == null) {
assertNull(actual);
return;
@ -79,6 +82,7 @@ public class ExtractorAsserts {
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected, actual);
// using new ArrayList<> to make sure the type is the same
assertEquals(new ArrayList<>(expected), new ArrayList<>(actual));
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java
View File

@ -66,7 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
@Nullable public String expectedDashMpdUrlContains() { return null; } // default: no dash mpd
public boolean expectedHasFrames() { return true; } // default: there are frames
public String expectedHost() { return ""; } // default: no host for centralized platforms
public String expectedPrivacy() { return ""; } // default: no privacy policy available
public StreamExtractor.Privacy expectedPrivacy() { return StreamExtractor.Privacy.PUBLIC; } // default: public
public String expectedCategory() { return ""; } // default: no category
public String expectedLicence() { return ""; } // default: no licence
public Locale expectedLanguageInfo() { return null; } // default: no language info available

188
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -5,15 +5,17 @@ import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
import java.util.Locale;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -42,22 +44,54 @@ public class MediaCCCStreamExtractorTest {
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "gpn18"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); }
@Override public long expectedLength() { return 3097; }
@Override public long expectedViewCountAtLeast() { return 2380; }
@Nullable @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Nullable
@Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; }
@Nullable
@Override
public String expectedTextualUploadDate() {
return "2018-05-11T02:00:00.000+02:00";
}
@Override
public long expectedLikeCountAtLeast() {
return -1;
}
@Override
public long expectedDislikeCountAtLeast() {
return -1;
}
@Override
public boolean expectedHasRelatedStreams() {
return false;
}
@Override
public boolean expectedHasSubtitles() {
return false;
}
@Override
public boolean expectedHasFrames() {
return false;
}
@Override
public List<String> expectedTags() {
return Arrays.asList("gpn18", "105");
}
@Override
public int expectedStreamSegmentsCount() {
return 0;
}
@Override
@Test
@ -86,6 +120,11 @@ public class MediaCCCStreamExtractorTest {
super.testAudioStreams();
assertEquals(2, extractor.getAudioStreams().size());
}
@Override
public Locale expectedLanguageInfo() {
return new Locale("de");
}
}
public static class _36c3PrivacyMessaging extends DefaultStreamExtractorTest {
@ -100,27 +139,107 @@ public class MediaCCCStreamExtractorTest {
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return "What's left for private messaging?"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override
public StreamExtractor extractor() {
return extractor;
}
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "36c3"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); }
@Override public long expectedLength() { return 3603; }
@Override public long expectedViewCountAtLeast() { return 2380; }
@Nullable @Override public String expectedUploadDate() { return "2020-01-11 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); }
@Override
public StreamingService expectedService() {
return MediaCCC;
}
@Override
public String expectedName() {
return "What's left for private messaging?";
}
@Override
public String expectedId() {
return ID;
}
@Override
public String expectedUrlContains() {
return URL;
}
@Override
public String expectedOriginalUrlContains() {
return URL;
}
@Override
public StreamType expectedStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public String expectedUploaderName() {
return "36c3";
}
@Override
public String expectedUploaderUrl() {
return "https://media.ccc.de/c/36c3";
}
@Override
public List<String> expectedDescriptionContains() {
return Arrays.asList("WhatsApp", "Signal");
}
@Override
public long expectedLength() {
return 3603;
}
@Override
public long expectedViewCountAtLeast() {
return 2380;
}
@Nullable
@Override
public String expectedUploadDate() {
return "2020-01-11 00:00:00.000";
}
@Nullable
@Override
public String expectedTextualUploadDate() {
return "2020-01-11T01:00:00.000+01:00";
}
@Override
public long expectedLikeCountAtLeast() {
return -1;
}
@Override
public long expectedDislikeCountAtLeast() {
return -1;
}
@Override
public boolean expectedHasRelatedStreams() {
return false;
}
@Override
public boolean expectedHasSubtitles() {
return false;
}
@Override
public boolean expectedHasFrames() {
return false;
}
@Override
public List<String> expectedTags() {
return Arrays.asList("36c3", "10565", "2019", "Security", "Main");
}
@Override
@Test
@ -149,5 +268,10 @@ public class MediaCCCStreamExtractorTest {
super.testAudioStreams();
assertEquals(2, extractor.getAudioStreams().size());
}
@Override
public Locale expectedLanguageInfo() {
return new Locale("en");
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java
View File

@ -88,7 +88,6 @@ public class PeertubeStreamExtractorTest {
@Override public boolean expectedHasAudioStreams() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "framatube.org"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@ -139,7 +138,6 @@ public class PeertubeStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "nocensoring.net"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Art"; }
@Override public String expectedLicence() { return "Attribution"; }
@Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }

25
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View File

@ -34,11 +34,15 @@ public class SoundcloudStreamExtractorTest {
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
@Test(expected = GeographicRestrictionException.class)
public void geoRestrictedContent() throws Exception {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getStreamExtractor(URL);
try {
extractor.fetchPage();
} catch (final GeographicRestrictionException e) {
// expected
}
}
@Override public StreamExtractor extractor() { return extractor; }
@ -67,6 +71,8 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public String expectedLicence() { return "all-rights-reserved"; }
@Override public String expectedCategory() { return "Pop"; }
}
public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest {
@ -76,11 +82,15 @@ public class SoundcloudStreamExtractorTest {
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
@Test(expected = SoundCloudGoPlusContentException.class)
public void goPlusContent() throws Exception {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getStreamExtractor(URL);
try {
extractor.fetchPage();
} catch (final SoundCloudGoPlusContentException e) {
// expected
}
}
@Override public StreamExtractor extractor() { return extractor; }
@ -109,6 +119,8 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return "all-rights-reserved"; }
@Override public String expectedCategory() { return "Dance"; }
}
public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
@ -148,6 +160,11 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return "cc-by"; }
@Override public String expectedCategory() { return "Podcast"; }
@Override public List<String> expectedTags() {
return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia");
}
@Override
@Test

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -54,4 +54,16 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Override public int expectedAgeLimit() { return 18; }
@Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public String expectedCategory() {return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override
public List<String> expectedTags() {
return Arrays.asList("AEE", "AEE 2017", "AVN", "AVN 2016", "AVN 2017", "AVN 2017 Expo In Las Vegas",
"AVN Awards Show", "AVN Expo", "AVN Las Vegas", "AVN Magazine", "AVN Vlog", "Ariana Marie",
"August Ames", "Brenna Sparks", "CeCe Capella", "Cindy Starfall", "Elsa Jean", "Emma Hix",
"FINGERING", "FINGERING P0RNSTARS", "FINGERING PORNSTARS", "Kaho Shibuya", "Keisha Grey",
"Kimberly Chi", "Las Vegas", "Mia Martinez", "Pornstar", "Pornstars", "Riley Reid",
"Samantha Saint", "Vegas", "Vicki Chase");
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@ -56,4 +56,8 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
@Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
@Override public long expectedLikeCountAtLeast() { return 13300; }
@Override public long expectedDislikeCountAtLeast() { return 2600; }
@Override public List<String> expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); }
@Override public String expectedCategory() { return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
}

83
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -15,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
@ -56,6 +58,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeStreamExtractorDefaultTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String BASE_URL = "https://www.youtube.com/watch?v=";
public static final String YOUTUBE_LICENCE = "YouTube licence";
public static class NotAvailable {
@BeforeClass
@ -145,6 +148,8 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Entertainment"; }
// @formatter:on
}
@ -185,6 +190,16 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
@Override public boolean expectedUploaderVerified() { return true; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public List<String> expectedTags() {
return Arrays.asList("2018", "8 plus", "apple", "apple iphone", "apple iphone x", "best", "best android",
"best smartphone", "cool gadgets", "find", "find x", "find x review", "find x unboxing", "findx",
"galaxy s9", "galaxy s9+", "hands on", "iphone 8", "iphone 8 plus", "iphone x", "new iphone", "nex",
"oneplus 6", "oppo", "oppo find x", "oppo find x hands on", "oppo find x review",
"oppo find x unboxing", "oppo findx", "pixel 2 xl", "review", "samsung", "samsung galaxy",
"samsung galaxy s9", "smartphone", "unbox therapy", "unboxing", "vivo", "vivo apex", "vivo nex");
}
// @formatter:on
}
@ -260,8 +275,16 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 32100; }
@Override public long expectedDislikeCountAtLeast() { return 750; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public int expectedStreamSegmentsCount() { return 17; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Music"; }
@Override public List<String> expectedTags() {
return Arrays.asList("2019", "2019 anime", "Anime OST", "Epic anime ost", "OST Anime",
"anime epic soundtrack", "armin", "attack on titan", "battle anime ost", "battle anime soundtracks",
"combat anime ost", "epic soundtrack", "eren", "mikasa", "motivational anime ost",
"motivational anime soundtracks", "shingeki no kyojin");
}
// @formatter:on
@Test
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(3);
@ -270,7 +293,6 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
// @formatter:on
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@ -308,6 +330,16 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 20000; }
@Override public boolean expectedHasSubtitles() { return true; }
@Override public int expectedStreamSegmentsCount() { return 7; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public List<String> expectedTags() {
return Arrays.asList("Diabetes", "Erkältung", "Gesundheit", "Immunabwehr", "Immunsystem", "Infektion",
"Komisch alles chemisch", "Krebs", "Lab", "Lesch", "Mai", "Mai Thi", "Mai Thi Nguyen-Kim",
"Mangel", "Nahrungsergänzungsmittel", "Nguyen", "Nguyen Kim", "Nguyen-Kim", "Quarks", "Sommer",
"Supplemente", "Supplements", "Tabletten", "Terra X", "TerraX", "The Secret Life Of Scientists",
"Tropfen", "Vitamin D", "Vitamin-D-Mangel", "Vitamine", "Winter", "einnehmen", "maiLab", "nehmen",
"supplementieren", "Überdosis", "Überschuss");
}
// @formatter:on
@Test
@ -369,7 +401,54 @@ public class YoutubeStreamExtractorDefaultTest {
));
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Education"; }
@Override public List<String> expectedTags() {
return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus",
"Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf",
"Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial",
"Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale",
"Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische",
"Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale");
}
// @formatter:on
}
public static class UnlistedTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
extractor.fetchPage();
}
@Test
public void testGetUnlisted() {
assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy());
}
}
public static class CCLicensed {
// StreamSegment example with macro-makers panel and transcription panel
private static final String ID = "M4gD1WSo5mA";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetLicence() throws ParsingException {
assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence());
}
}
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@ -39,13 +39,13 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
@Override public String expectedUploaderName() { return "ChilledCow"; }
@Override public String expectedUploaderName() { return "Lofi Girl"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://bit.ly/chilledcow-playlists",
"https://bit.ly/chilledcow-submissions");
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public boolean expectedUploaderVerified() { return false; }
@Override public long expectedLength() { return 0; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 0; }
@ -56,4 +56,14 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@Override public boolean expectedHasSubtitles() { return false; }
@Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override public String expectedCategory() { return "Music"; }
@Override public List<String> expectedTags() {
return Arrays.asList("beats to relax", "chilled cow", "chilled cow radio", "chilledcow", "chilledcow radio",
"chilledcow station", "chillhop", "hip hop", "hiphop", "lo fi", "lo fi hip hop", "lo fi hip hop radio",
"lo fi hiphop", "lo fi radio", "lo-fi", "lo-fi hip hop", "lo-fi hip hop radio", "lo-fi hiphop",
"lo-fi radio", "lofi", "lofi hip hop", "lofi hip hop radio", "lofi hiphop", "lofi radio", "music",
"lofi radio chilledcow", "music to study", "playlist", "radio", "relaxing music", "study music",
"lofi hip hop radio - beats to relax\\/study to");
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View File

@ -15,6 +15,7 @@ import java.util.List;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED;
public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
@ -50,4 +51,8 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
@Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; }
@Override public long expectedLikeCountAtLeast() { return 110; }
@Override public long expectedDislikeCountAtLeast() { return 0; }
@Override public StreamExtractor.Privacy expectedPrivacy() { return UNLISTED; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override public String expectedCategory() { return "Gaming"; }
@Override public List<String> expectedTags() { return Arrays.asList("dark souls", "hooked", "praise the casual"); }
}