Merge pull request #239 from B0pol/peertube

[PeerTube] added metadata, fix descriptions, fix thumbnail, fix upload date, fix age limit, update tests.
This commit is contained in:
Tobias Groza 2020-02-08 22:31:02 +01:00 committed by GitHub
commit 3add704f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 565 additions and 143 deletions

View File

@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
## Supported sites ## Supported sites

View File

@ -17,6 +17,7 @@ import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor { public class MediaCCCStreamExtractor extends StreamExtractor {
@ -47,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public String getDescription() throws ParsingException { public Description getDescription() throws ParsingException {
return data.getString("description"); return new Description(data.getString("description"), Description.PLAIN_TEXT);
} }
@Override @Override
@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getOriginalUrl() throws ParsingException { public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link"); return data.getString("frontend_link");
} }
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
} }

View File

@ -4,6 +4,7 @@ import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.TimeZone;
import org.jsoup.helper.StringUtil; import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -12,21 +13,23 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
public class PeertubeParsingHelper { public class PeertubeParsingHelper {
private PeertubeParsingHelper() { private PeertubeParsingHelper() {
} }
public static void validate(JsonObject json) throws ContentNotAvailableException { public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error"); String error = json.getString("error");
if(!StringUtil.isBlank(error)) { if (!StringUtil.isBlank(error)) {
throw new ContentNotAvailableException(error); throw new ContentNotAvailableException(error);
} }
} }
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date; Date date;
try { try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) { } catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
} }

View File

@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
try { try {
Document doc = Jsoup.parse(htmlText); Document doc = Jsoup.parse(htmlText);
return doc.body().text(); return doc.body().text();
}catch(Exception e) { } catch(Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
} }
} }
@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
String value; String value;
try { try {
value = JsonUtils.getString(item, "account.avatar.path"); value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) { } catch(Exception e) {
value = "/client/assets/images/default-avatar.png"; value = "/client/assets/images/default-avatar.png";
} }
return baseUrl + value; return baseUrl + value;
@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override @Override
public String getAuthorName() throws ParsingException { public String getAuthorName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName"); return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
} }
@Override @Override

View File

@ -6,10 +6,11 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale;
import org.jsoup.helper.StringUtil; import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
@ -20,34 +21,28 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import javax.annotation.Nonnull;
public class PeertubeStreamExtractor extends StreamExtractor { public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json; private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>(); private List<SubtitlesStream> subtitles = new ArrayList<>();
private final String baseUrl;
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler); super(service, linkHandler);
this.baseUrl = getBaseUrl(); this.baseUrl = getBaseUrl();
} }
@Override @Override
public String getTextualUploadDate() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(json, "publishedAt"); return JsonUtils.getString(json, "publishedAt");
@ -63,24 +58,42 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
} }
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "thumbnailPath"); return baseUrl + JsonUtils.getString(json, "previewPath");
} }
@Override @Override
public String getDescription() throws ParsingException { public Description getDescription() throws ParsingException {
String text;
try { try {
return JsonUtils.getString(json, "description"); text = JsonUtils.getString(json, "description");
}catch(ParsingException e) { } catch (ParsingException e) {
return "No description"; return Description.emptyDescription;
} }
if (text.length() == 250 && text.substring(247).equals("...")) {
//if description is shortened, get full description
Downloader dl = NewPipe.getDownloader();
try {
Response response = dl.get(getUrl() + "/description");
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
}
}
return new Description(text, Description.MARKDOWN);
} }
@Override @Override
public int getAgeLimit() throws ParsingException { public int getAgeLimit() throws ParsingException {
return NO_AGE_LIMIT; boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
if (isNSFW) {
return 18;
} else {
return NO_AGE_LIMIT;
}
} }
@Override @Override
@ -130,7 +143,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
String value; String value;
try { try {
value = JsonUtils.getString(json, "account.avatar.path"); value = JsonUtils.getString(json, "account.avatar.path");
}catch(Exception e) { } catch (Exception e) {
value = "/client/assets/images/default-avatar.png"; value = "/client/assets/images/default-avatar.png";
} }
return baseUrl + value; return baseUrl + value;
@ -157,8 +170,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
List<VideoStream> videoStreams = new ArrayList<>(); List<VideoStream> videoStreams = new ArrayList<>();
try { try {
JsonArray streams = json.getArray("files", new JsonArray()); JsonArray streams = json.getArray("files", new JsonArray());
for(Object s: streams) { for (Object s : streams) {
if(!(s instanceof JsonObject)) continue; if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s; JsonObject stream = (JsonObject) s;
String url = JsonUtils.getString(stream, "fileUrl"); String url = JsonUtils.getString(stream, "fileUrl");
String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
@ -192,8 +205,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override @Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
List<SubtitlesStream> filteredSubs = new ArrayList<>(); List<SubtitlesStream> filteredSubs = new ArrayList<>();
for(SubtitlesStream sub: subtitles) { for (SubtitlesStream sub : subtitles) {
if(sub.getFormat() == format) { if (sub.getFormat() == format) {
filteredSubs.add(sub); filteredSubs.add(sub);
} }
} }
@ -215,29 +228,40 @@ public class PeertubeStreamExtractor extends StreamExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
List<String> tags = getTags(); List<String> tags = getTags();
String apiUrl = null; String apiUrl = null;
if(!tags.isEmpty()) { if (!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags); apiUrl = getRelatedStreamsUrl(tags);
}else { } else {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
} }
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector; return collector;
} }
private List<String> getTags(){ @Override
public List<String> getTags() {
try { try {
return (List) JsonUtils.getArray(json, "tags"); return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) { } catch (Exception e) {
return Collections.emptyList(); return Collections.emptyList();
} }
} }
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
try {
return JsonUtils.getString(json, "support");
} catch (ParsingException e) {
return "";
}
}
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException { private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
StringBuilder params = new StringBuilder(); StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt"); params.append("start=0&count=8&sort=-createdAt");
for(String tag : tags) { for (String tag : tags) {
params.append("&tagsOneOf="); params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8")); params.append(URLEncoder.encode(tag, "UTF-8"));
} }
@ -247,38 +271,38 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl); Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null; JsonObject relatedVideosJson = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) { if (null != response && !StringUtil.isBlank(response.responseBody())) {
try { try {
relatedVideosJson = JsonParser.object().from(response.responseBody()); relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for related videos", e); throw new ParsingException("Could not parse json data for related videos", e);
} }
} }
if(relatedVideosJson != null) { if (relatedVideosJson != null) {
collectStreamsFrom(collector, relatedVideosJson); collectStreamsFrom(collector, relatedVideosJson);
} }
} }
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
JsonArray contents; JsonArray contents;
try { try {
contents = (JsonArray) JsonUtils.getValue(json, "data"); contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) { } catch (Exception e) {
throw new ParsingException("unable to extract related videos", e); throw new ParsingException("unable to extract related videos", e);
} }
for(Object c: contents) { for (Object c : contents) {
if(c instanceof JsonObject) { if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c; final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams //do not add the same stream in related streams
if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
} }
} }
} }
@Override @Override
public String getErrorMessage() { public String getErrorMessage() {
@ -288,12 +312,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override @Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl()); Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) { if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody()); setInitialData(response.responseBody());
}else { } else {
throw new ExtractionException("Unable to extract peertube channel data"); throw new ExtractionException("Unable to extract peertube channel data");
} }
loadSubtitles(); loadSubtitles();
} }
@ -303,24 +327,25 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} catch (JsonParserException e) { } catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube stream data", e); throw new ExtractionException("Unable to extract peertube stream data", e);
} }
if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); if (null == json) throw new ExtractionException("Unable to extract peertube stream data");
PeertubeParsingHelper.validate(json); PeertubeParsingHelper.validate(json);
} }
private void loadSubtitles() { private void loadSubtitles() {
if (subtitles.isEmpty()) { if (subtitles.isEmpty()) {
try { try {
Response response = getDownloader().get(getUrl() + "/captions"); Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data"); JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for(Object c: captions) { for (Object c : captions) {
if(c instanceof JsonObject) { if (c instanceof JsonObject) {
JsonObject caption = (JsonObject)c; JsonObject caption = (JsonObject) c;
String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String url = baseUrl + JsonUtils.getString(caption, "captionPath");
String languageCode = JsonUtils.getString(caption, "language.id"); String languageCode = JsonUtils.getString(caption, "language.id");
String ext = url.substring(url.lastIndexOf(".") + 1); String ext = url.substring(url.lastIndexOf(".") + 1);
MediaFormat fmt = MediaFormat.getFromSuffix(ext); MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); if (fmt != null && languageCode != null)
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
} }
} }
} catch (Exception e) { } catch (Exception e) {
@ -339,4 +364,32 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + "/videos/watch/" + getId(); return baseUrl + "/videos/watch/" + getId();
} }
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
}
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
}
@Override
public String getCategory() throws ParsingException {
return JsonUtils.getString(json, "category.label");
}
@Override
public String getLicence() throws ParsingException {
return JsonUtils.getString(json, "licence.label");
}
@Override
public Locale getLanguageInfo() throws ParsingException {
try {
return new Locale(JsonUtils.getString(json, "language.id"));
} catch (ParsingException e) {
return null;
}
}
} }

View File

@ -20,6 +20,7 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale;
public class SoundcloudStreamExtractor extends StreamExtractor { public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track; private JsonObject track;
@ -73,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return artworkUrlBetterResolution; return artworkUrlBetterResolution;
} }
@Nonnull
@Override @Override
public String getDescription() { public Description getDescription() {
return track.getString("description"); return new Description(track.getString("description"), Description.PLAIN_TEXT);
} }
@Override @Override
@ -254,4 +254,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getErrorMessage() { public String getErrorMessage() {
return null; return null;
} }
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
} }

View File

@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public String getDescription() throws ParsingException { public Description getDescription() throws ParsingException {
assertPageFetched(); assertPageFetched();
try { try {
// first try to get html-formatted description // first try to get html-formatted description
return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
} catch (Exception e) { } catch (Exception e) {
try { try {
// fallback to raw non-html description // fallback to raw non-html description
return playerResponse.getObject("videoDetails").getString("shortDescription"); return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
} catch (Exception ignored) { } catch (Exception ignored) {
throw new ParsingException("Could not get the description", e); throw new ParsingException("Could not get the description", e);
} }
@ -1134,4 +1134,41 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ExtractionException(e); throw new ExtractionException(e);
} }
} }
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
} }

View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
private String content;
private int type;
public Description(String content, int type) {
this.type = type;
if (content == null) {
this.content = "";
} else {
this.content = content;
}
}
public String getContent() {
return content;
}
public int getType() {
return type;
}
}

View File

@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -34,6 +35,7 @@ import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale;
/** /**
* Scrapes information from a video/audio streaming service (eg, YouTube). * Scrapes information from a video/audio streaming service (eg, YouTube).
@ -82,12 +84,12 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getThumbnailUrl() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException;
/** /**
* This is the stream description. On YouTube this is the video description. You can return simple HTML here. * This is the stream description.
* @return The description of the stream/video. * @return The description of the stream/video or Description.emptyDescription if the description is empty.
* @throws ParsingException * @throws ParsingException
*/ */
@Nonnull @Nonnull
public abstract String getDescription() throws ParsingException; public abstract Description getDescription() throws ParsingException;
/** /**
* Get the age limit. * Get the age limit.
@ -349,4 +351,74 @@ public abstract class StreamExtractor extends Extractor {
return 0; return 0;
} }
} }
/**
* The host of the stream (Eg. peertube.cpy.re).
* If the host is not available, or if the service doesn't use
* a federated system, but a centralised system,
* you can simply return an empty string.
* @return the host of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getHost() throws ParsingException;
/**
* The privacy of the stream (Eg. Public, Private, Unlisted).
* If the privacy is not available you can simply return an empty string.
* @return the privacy of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getPrivacy() throws ParsingException;
/**
* The name of the category of the stream.
* If the category is not available you can simply return an empty string.
* @return the category of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getCategory() throws ParsingException;
/**
* The name of the licence of the stream.
* If the licence is not available you can simply return an empty string.
* @return the licence of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getLicence() throws ParsingException;
/**
* The locale language of the stream.
* If the language is not available you can simply return null.
* If the language is provided by a language code, you can return
* new Locale(language_code);
* @return the locale language of the stream or null.
* @throws ParsingException
*/
@Nullable
public abstract Locale getLanguageInfo() throws ParsingException;
/**
* The list of tags of the stream.
* If the tag list is not available you can simply return an empty list.
* @return the list of tags of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<String> getTags() throws ParsingException;
/**
* The support information of the stream.
* see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
* (support button).
* If the support information are not available,
* you can simply return an empty String.
* @return the support information of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getSupportInfo() throws ParsingException;
} }

View File

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Locale;
/* /*
* Created by Christian Schabesberger on 26.08.15. * Created by Christian Schabesberger on 26.08.15.
@ -270,6 +271,43 @@ public class StreamInfo extends Info {
streamInfo.addError(e); streamInfo.addError(e);
} }
//additional info
try {
streamInfo.setHost(extractor.getHost());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setPrivacy(extractor.getPrivacy());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setCategory(extractor.getCategory());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLicence(extractor.getLicence());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLanguageInfo(extractor.getLanguageInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTags(extractor.getTags());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setSupportInfo(extractor.getSupportInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
return streamInfo; return streamInfo;
@ -281,7 +319,7 @@ public class StreamInfo extends Info {
private DateWrapper uploadDate; private DateWrapper uploadDate;
private long duration = -1; private long duration = -1;
private int ageLimit = -1; private int ageLimit = -1;
private String description; private Description description;
private long viewCount = -1; private long viewCount = -1;
private long likeCount = -1; private long likeCount = -1;
@ -308,6 +346,14 @@ public class StreamInfo extends Info {
private long startPosition = 0; private long startPosition = 0;
private List<SubtitlesStream> subtitles = new ArrayList<>(); private List<SubtitlesStream> subtitles = new ArrayList<>();
private String host = "";
private String privacy = "";
private String category = "";
private String licence = "";
private String support = "";
private Locale language = null;
private List<String> tags = new ArrayList<>();
/** /**
* Get the stream type * Get the stream type
* *
@ -371,11 +417,11 @@ public class StreamInfo extends Info {
this.ageLimit = ageLimit; this.ageLimit = ageLimit;
} }
public String getDescription() { public Description getDescription() {
return description; return description;
} }
public void setDescription(String description) { public void setDescription(Description description) {
this.description = description; this.description = description;
} }
@ -533,4 +579,59 @@ public class StreamInfo extends Info {
this.subtitles = subtitles; this.subtitles = subtitles;
} }
public String getHost() {
return this.host;
}
public void setHost(String str) {
this.host = str;
}
public String getPrivacy() {
return this.privacy;
}
public void setPrivacy(String str) {
this.privacy = str;
}
public String getCategory() {
return this.category;
}
public void setCategory(String cat) {
this.category = cat;
}
public String getLicence() {
return this.licence;
}
public void setLicence(String str) {
this.licence = str;
}
public Locale getLanguageInfo() {
return this.language;
}
public void setLanguageInfo(Locale lang) {
this.language = lang;
}
public List<String> getTags() {
return this.tags;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
public void setSupportInfo(String support) {
this.support = support;
}
public String getSupportInfo() {
return this.support;
}
} }

View File

@ -16,61 +16,71 @@ public class JsonUtils {
private JsonUtils() { private JsonUtils() {
} }
@Nonnull @Nonnull
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
List<String> keys = Arrays.asList(path.split("\\.")); List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1)); object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path); if (null == object) throw new ParsingException("Unable to get " + path);
Object result = object.get(keys.get(keys.size() - 1)); Object result = object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path); if (null == result) throw new ParsingException("Unable to get " + path);
return result; return result;
} }
@Nonnull @Nonnull
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof String) { if (value instanceof String) {
return (String) value; return (String) value;
}else { } else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
} }
@Nonnull @Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof Number) { if (value instanceof Boolean) {
return (Boolean) value;
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if (value instanceof Number) {
return (Number) value; return (Number) value;
}else { } else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
} }
@Nonnull @Nonnull
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof JsonObject) { if (value instanceof JsonObject) {
return (JsonObject) value; return (JsonObject) value;
}else { } else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
} }
@Nonnull @Nonnull
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof JsonArray) { if (value instanceof JsonArray) {
return (JsonArray) value; return (JsonArray) value;
}else { } else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
} }
@Nonnull @Nonnull
public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<Object> result = new ArrayList<>(); List<Object> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) { for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i); JsonObject obj = array.getObject(i);

View File

@ -185,5 +185,4 @@ public class Utils {
} }
return uri.getProtocol() + "://" + uri.getAuthority(); return uri.getProtocol() + "://" + uri.getAuthority();
} }
} }

View File

@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
} }
@Test @Test
@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
@Test @Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false; boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName())); assertTrue("Comments".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");

View File

@ -11,6 +11,8 @@ import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Locale;
import java.util.TimeZone;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Ignore; import org.junit.Ignore;
@ -29,16 +31,28 @@ import org.schabi.newpipe.extractor.stream.StreamType;
*/ */
public class PeertubeStreamExtractorDefaultTest { public class PeertubeStreamExtractorDefaultTest {
private static PeertubeStreamExtractor extractor; private static PeertubeStreamExtractor extractor;
private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
@BeforeClass @BeforeClass
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel // setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
extractor.fetchPage(); extractor.fetchPage();
} }
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test @Test
public void testGetInvalidTimeStamp() throws ParsingException { public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "", assertTrue(extractor.getTimeStamp() + "",
@ -47,22 +61,37 @@ public class PeertubeStreamExtractorDefaultTest {
@Test @Test
public void testGetTitle() throws ParsingException { public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Power Corrupts the Best"); assertEquals("What is PeerTube?", extractor.getName());
} }
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetLargeDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
}
@Test
public void testGetEmptyDescription() throws Exception {
PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
extractorEmpty.fetchPage();
assertEquals("", extractorEmpty.getDescription().getContent());
}
@Test
public void testGetSmallDescription() throws Exception {
PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
extractorSmall.fetchPage();
assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
} }
@Test @Test
public void testGetUploaderName() throws ParsingException { public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "Rowsedower"); assertEquals("Framasoft", extractor.getUploaderName());
} }
@Test @Test
public void testGetLength() throws ParsingException { public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 269); assertEquals(113, extractor.getLength());
} }
@Test @Test
@ -71,18 +100,10 @@ public class PeertubeStreamExtractorDefaultTest {
extractor.getViewCount() > 10); extractor.getViewCount() > 10);
} }
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test @Test
public void testGetUploaderUrl() throws ParsingException { public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl()); assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
} }
@Test @Test
@ -115,11 +136,31 @@ public class PeertubeStreamExtractorDefaultTest {
@Test @Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException { public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty()); assertFalse(extractor.getSubtitlesDefault().isEmpty());
} }
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty()); assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetAgeLimit() throws ExtractionException, IOException {
assertEquals(0, extractor.getAgeLimit());
PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7");
ageLimit.fetchPage();
assertEquals(18, ageLimit.getAgeLimit());
}
@Test
public void testGetSupportInformation() throws ExtractionException, IOException {
PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
supportInfoExtractor.fetchPage();
assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
}
@Test
public void testGetLanguageInformation() throws ParsingException {
assertEquals(new Locale("en"), extractor.getLanguageInfo());
} }
} }

View File

@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertEquals("The Perfect LUV Tape®", extractor.getDescription()); assertEquals("The Perfect LUV Tape®", extractor.getDescription().getContent());
} }
@Test @Test

View File

@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test

View File

@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test

View File

@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("http://adele.com")); assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi..."));
} }
@Test @Test
@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub..."));
assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8..."));
assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/..."));
} }
} }
@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
} }
} }

View File

@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }
@Test @Test
public void testGetFullLinksInDescription() throws ParsingException { public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/"));
assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba..."));
} }
@Test @Test