Merge pull request #239 from B0pol/peertube

[PeerTube] added metadata, fix descriptions, fix thumbnail, fix upload date, fix age limit, update tests.

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -17,6 +17,7 @@ import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 public class MediaCCCStreamExtractor extends StreamExtractor {
 
@@ -47,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
-        return data.getString("description");
+        return new Description(data.getString("description"), Description.PLAIN_TEXT);
     }
 
     @Override
@@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     public String getOriginalUrl() throws ParsingException {
         return data.getString("frontend_link");
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -4,6 +4,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.TimeZone;
 
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -18,7 +19,7 @@ public class PeertubeParsingHelper {
 
     public static void validate(JsonObject json) throws ContentNotAvailableException {
         String error = json.getString("error");
-        if(!StringUtil.isBlank(error)) {
+        if (!StringUtil.isBlank(error)) {
             throw new ContentNotAvailableException(error);
         }
     }
@@ -26,7 +27,9 @@ public class PeertubeParsingHelper {
     public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
         Date date;
         try {
-            date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
+            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
+            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+            date = sdf.parse(textualUploadDate);
         } catch (ParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         try {
             Document doc = Jsoup.parse(htmlText);
             return doc.body().text();
-        }catch(Exception e) {
+        } catch(Exception e) {
             return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
         }
     }
@@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         String value;
         try {
             value = JsonUtils.getString(item, "account.avatar.path");
-        }catch(Exception e) {
+        } catch(Exception e) {
             value = "/client/assets/images/default-avatar.png";
         }
         return baseUrl + value;
@@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
 
     @Override
     public String getAuthorName() throws ParsingException {
-        return JsonUtils.getString(item, "account.displayName");
+        return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -6,10 +6,11 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -20,28 +21,22 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItem;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.SubtitlesStream;
-import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
+import javax.annotation.Nonnull;
+
 public class PeertubeStreamExtractor extends StreamExtractor {
 
 
+    private final String baseUrl;
     private JsonObject json;
     private List<SubtitlesStream> subtitles = new ArrayList<>();
-    private final String baseUrl;
 
     public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
         super(service, linkHandler);
@@ -66,22 +61,40 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        return baseUrl + JsonUtils.getString(json, "thumbnailPath");
+        return baseUrl + JsonUtils.getString(json, "previewPath");
     }
 
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
+        String text;
         try {
-            return JsonUtils.getString(json, "description");
+            text = JsonUtils.getString(json, "description");
-        }catch(ParsingException e) {
+        } catch (ParsingException e) {
-            return "No description";
+            return Description.emptyDescription;
         }
+        if (text.length() == 250 && text.substring(247).equals("...")) {
+            //if description is shortened, get full description
+            Downloader dl = NewPipe.getDownloader();
+            try {
+                Response response = dl.get(getUrl() + "/description");
+                JsonObject jsonObject = JsonParser.object().from(response.responseBody());
+                text = JsonUtils.getString(jsonObject, "description");
+            } catch (ReCaptchaException | IOException | JsonParserException e) {
+                e.printStackTrace();
+            }
+        }
+        return new Description(text, Description.MARKDOWN);
     }
 
     @Override
     public int getAgeLimit() throws ParsingException {
+        boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
+        if (isNSFW) {
+            return 18;
+        } else {
             return NO_AGE_LIMIT;
         }
+    }
 
     @Override
     public long getLength() throws ParsingException {
@@ -130,7 +143,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         String value;
         try {
             value = JsonUtils.getString(json, "account.avatar.path");
-        }catch(Exception e) {
+        } catch (Exception e) {
             value = "/client/assets/images/default-avatar.png";
         }
         return baseUrl + value;
@@ -157,8 +170,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         List<VideoStream> videoStreams = new ArrayList<>();
         try {
             JsonArray streams = json.getArray("files", new JsonArray());
-            for(Object s: streams) {
+            for (Object s : streams) {
-                if(!(s instanceof JsonObject)) continue;
+                if (!(s instanceof JsonObject)) continue;
                 JsonObject stream = (JsonObject) s;
                 String url = JsonUtils.getString(stream, "fileUrl");
                 String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
@@ -192,8 +205,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Override
     public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
         List<SubtitlesStream> filteredSubs = new ArrayList<>();
-        for(SubtitlesStream sub: subtitles) {
+        for (SubtitlesStream sub : subtitles) {
-            if(sub.getFormat() == format) {
+            if (sub.getFormat() == format) {
                 filteredSubs.add(sub);
             }
         }
@@ -215,17 +228,18 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         List<String> tags = getTags();
         String apiUrl = null;
-        if(!tags.isEmpty()) {
+        if (!tags.isEmpty()) {
             apiUrl = getRelatedStreamsUrl(tags);
 
-        }else {
+        } else {
             apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
         }
-        if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
+        if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
         return collector;
     }
 
-    private List<String> getTags(){
+    @Override
+    public List<String> getTags() {
         try {
             return (List) JsonUtils.getArray(json, "tags");
         } catch (Exception e) {
@@ -233,11 +247,21 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        try {
+            return JsonUtils.getString(json, "support");
+        } catch (ParsingException e) {
+            return "";
+        }
+    }
+
     private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
         String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
         StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");
-        for(String tag : tags) {
+        for (String tag : tags) {
             params.append("&tagsOneOf=");
             params.append(URLEncoder.encode(tag, "UTF-8"));
         }
@@ -247,7 +271,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
         Response response = getDownloader().get(apiUrl);
         JsonObject relatedVideosJson = null;
-        if(null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (null != response && !StringUtil.isBlank(response.responseBody())) {
             try {
                 relatedVideosJson = JsonParser.object().from(response.responseBody());
             } catch (JsonParserException e) {
@@ -255,7 +279,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
             }
         }
 
-        if(relatedVideosJson != null) {
+        if (relatedVideosJson != null) {
             collectStreamsFrom(collector, relatedVideosJson);
         }
     }
@@ -264,16 +288,16 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         JsonArray contents;
         try {
             contents = (JsonArray) JsonUtils.getValue(json, "data");
-        }catch(Exception e) {
+        } catch (Exception e) {
             throw new ParsingException("unable to extract related videos", e);
         }
 
-        for(Object c: contents) {
+        for (Object c : contents) {
-            if(c instanceof JsonObject) {
+            if (c instanceof JsonObject) {
                 final JsonObject item = (JsonObject) c;
                 PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
                 //do not add the same stream in related streams
-                if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
+                if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
             }
         }
 
@@ -288,9 +312,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
         Response response = downloader.get(getUrl());
-        if(null != response && null != response.responseBody()) {
+        if (null != response && null != response.responseBody()) {
             setInitialData(response.responseBody());
-        }else {
+        } else {
             throw new ExtractionException("Unable to extract peertube channel data");
         }
 
@@ -303,7 +327,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         } catch (JsonParserException e) {
             throw new ExtractionException("Unable to extract peertube stream data", e);
         }
-        if(null == json) throw new ExtractionException("Unable to extract peertube stream data");
+        if (null == json) throw new ExtractionException("Unable to extract peertube stream data");
         PeertubeParsingHelper.validate(json);
     }
 
@@ -313,14 +337,15 @@ public class PeertubeStreamExtractor extends StreamExtractor {
                 Response response = getDownloader().get(getUrl() + "/captions");
                 JsonObject captionsJson = JsonParser.object().from(response.responseBody());
                 JsonArray captions = JsonUtils.getArray(captionsJson, "data");
-                for(Object c: captions) {
+                for (Object c : captions) {
-                    if(c instanceof JsonObject) {
+                    if (c instanceof JsonObject) {
-                        JsonObject caption = (JsonObject)c;
+                        JsonObject caption = (JsonObject) c;
                         String url = baseUrl + JsonUtils.getString(caption, "captionPath");
                         String languageCode = JsonUtils.getString(caption, "language.id");
                         String ext = url.substring(url.lastIndexOf(".") + 1);
                         MediaFormat fmt = MediaFormat.getFromSuffix(ext);
-                        if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
+                        if (fmt != null && languageCode != null)
+                            subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
                     }
                 }
             } catch (Exception e) {
@@ -339,4 +364,32 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return baseUrl + "/videos/watch/" + getId();
     }
 
+    @Override
+    public String getHost() throws ParsingException {
+        return JsonUtils.getString(json, "account.host");
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return JsonUtils.getString(json, "privacy.label");
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return JsonUtils.getString(json, "category.label");
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return JsonUtils.getString(json, "licence.label");
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        try {
+            return new Locale(JsonUtils.getString(json, "language.id"));
+        } catch (ParsingException e) {
+            return null;
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -20,6 +20,7 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
@@ -73,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return artworkUrlBetterResolution;
     }
 
-    @Nonnull
     @Override
-    public String getDescription() {
+    public Description getDescription() {
-        return track.getString("description");
+        return new Description(track.getString("description"), Description.PLAIN_TEXT);
     }
 
     @Override
@@ -254,4 +254,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     public String getErrorMessage() {
         return null;
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
         assertPageFetched();
         try {
             // first try to get html-formatted description
-            return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
+            return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
         } catch (Exception e) {
             try {
                 // fallback to raw non-html description
-                return playerResponse.getObject("videoDetails").getString("shortDescription");
+                return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
             } catch (Exception ignored) {
                 throw new ParsingException("Could not get the description", e);
             }
@@ -1134,4 +1134,41 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 			throw new ExtractionException(e);
 		}
 	}
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.stream;
+
+import java.io.Serializable;
+
+public class Description implements Serializable {
+
+    public static final int HTML = 1;
+    public static final int MARKDOWN = 2;
+    public static final int PLAIN_TEXT = 3;
+    public static final Description emptyDescription = new Description("", PLAIN_TEXT);
+
+    private String content;
+    private int type;
+
+    public Description(String content, int type) {
+        this.type = type;
+        if (content == null) {
+            this.content = "";
+        } else {
+            this.content = content;
+        }
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public int getType() {
+        return type;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
@@ -34,6 +35,7 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 /**
  * Scrapes information from a video/audio streaming service (eg, YouTube).
@@ -82,12 +84,12 @@ public abstract class StreamExtractor extends Extractor {
     public abstract String getThumbnailUrl() throws ParsingException;
 
     /**
-     * This is the stream description. On YouTube this is the video description. You can return simple HTML here.
+     * This is the stream description.
-     * @return The description of the stream/video.
+     * @return The description of the stream/video or Description.emptyDescription if the description is empty.
      * @throws ParsingException
      */
     @Nonnull
-    public abstract String getDescription() throws ParsingException;
+    public abstract Description getDescription() throws ParsingException;
 
     /**
      * Get the age limit.
@@ -349,4 +351,74 @@ public abstract class StreamExtractor extends Extractor {
             return 0;
         }
     }
+
+    /**
+     * The host of the stream (Eg. peertube.cpy.re).
+     * If the host is not available, or if the service doesn't use
+     * a federated system, but a centralised system,
+     * you can simply return an empty string.
+     * @return the host of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getHost() throws ParsingException;
+
+    /**
+     * The privacy of the stream (Eg. Public, Private, Unlisted…).
+     * If the privacy is not available you can simply return an empty string.
+     * @return the privacy of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getPrivacy() throws ParsingException;
+
+    /**
+     * The name of the category of the stream.
+     * If the category is not available you can simply return an empty string.
+     * @return the category of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getCategory() throws ParsingException;
+
+    /**
+     * The name of the licence of the stream.
+     * If the licence is not available you can simply return an empty string.
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getLicence() throws ParsingException;
+
+    /**
+     * The locale language of the stream.
+     * If the language is not available you can simply return null.
+     * If the language is provided by a language code, you can return
+     * new Locale(language_code);
+     * @return the locale language of the stream or null.
+     * @throws ParsingException
+     */
+    @Nullable
+    public abstract Locale getLanguageInfo() throws ParsingException;
+
+    /**
+     * The list of tags of the stream.
+     * If the tag list is not available you can simply return an empty list.
+     * @return the list of tags of the stream or an empty list.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract List<String> getTags() throws ParsingException;
+
+    /**
+     * The support information of the stream.
+     * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
+     * (support button).
+     * If the support information are not available,
+     * you can simply return an empty String.
+     * @return the support information of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getSupportInfo() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 /*
  * Created by Christian Schabesberger on 26.08.15.
@@ -270,6 +271,43 @@ public class StreamInfo extends Info {
             streamInfo.addError(e);
         }
 
+        //additional info
+        try {
+            streamInfo.setHost(extractor.getHost());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setPrivacy(extractor.getPrivacy());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setCategory(extractor.getCategory());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLicence(extractor.getLicence());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLanguageInfo(extractor.getLanguageInfo());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setTags(extractor.getTags());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setSupportInfo(extractor.getSupportInfo());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+
         streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
 
         return streamInfo;
@@ -281,7 +319,7 @@ public class StreamInfo extends Info {
     private DateWrapper uploadDate;
     private long duration = -1;
     private int ageLimit = -1;
-    private String description;
+    private Description description;
 
     private long viewCount = -1;
     private long likeCount = -1;
@@ -308,6 +346,14 @@ public class StreamInfo extends Info {
     private long startPosition = 0;
     private List<SubtitlesStream> subtitles = new ArrayList<>();
 
+    private String host = "";
+    private String privacy = "";
+    private String category = "";
+    private String licence = "";
+    private String support = "";
+    private Locale language = null;
+    private List<String> tags = new ArrayList<>();
+
     /**
      * Get the stream type
      *
@@ -371,11 +417,11 @@ public class StreamInfo extends Info {
         this.ageLimit = ageLimit;
     }
 
-    public String getDescription() {
+    public Description getDescription() {
         return description;
     }
 
-    public void setDescription(String description) {
+    public void setDescription(Description description) {
         this.description = description;
     }
 
@@ -533,4 +579,59 @@ public class StreamInfo extends Info {
         this.subtitles = subtitles;
     }
 
+    public String getHost() {
+        return this.host;
+    }
+
+    public void setHost(String str) {
+        this.host = str;
+    }
+
+    public String getPrivacy() {
+        return this.privacy;
+    }
+
+    public void setPrivacy(String str) {
+        this.privacy = str;
+    }
+
+    public String getCategory() {
+        return this.category;
+    }
+
+    public void setCategory(String cat) {
+        this.category = cat;
+    }
+
+    public String getLicence() {
+        return this.licence;
+    }
+
+    public void setLicence(String str) {
+        this.licence = str;
+    }
+
+    public Locale getLanguageInfo() {
+        return this.language;
+    }
+
+    public void setLanguageInfo(Locale lang) {
+        this.language = lang;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+    public void setSupportInfo(String support) {
+        this.support = support;
+    }
+
+    public String getSupportInfo() {
+        return this.support;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -18,52 +18,62 @@ public class JsonUtils {
     }
 
     @Nonnull
-    public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+    public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
 
         List<String> keys = Arrays.asList(path.split("\\."));
         object = getObject(object, keys.subList(0, keys.size() - 1));
         if (null == object) throw new ParsingException("Unable to get " + path);
         Object result = object.get(keys.get(keys.size() - 1));
-        if(null == result) throw new ParsingException("Unable to get " + path);
+        if (null == result) throw new ParsingException("Unable to get " + path);
         return result;
     }
 
     @Nonnull
-    public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+    public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
         Object value = getValue(object, path);
-        if(value instanceof String) {
+        if (value instanceof String) {
             return (String) value;
-        }else {
+        } else {
             throw new ParsingException("Unable to get " + path);
         }
     }
 
     @Nonnull
-    public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+    public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
         Object value = getValue(object, path);
-        if(value instanceof Number) {
+        if (value instanceof Boolean) {
+            return (Boolean) value;
+        } else {
+            throw new ParsingException("Unable to get " + path);
+        }
+    }
+
+    @Nonnull
+    public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
+        Object value = getValue(object, path);
+        if (value instanceof Number) {
             return (Number) value;
-        }else {
+        } else {
             throw new ParsingException("Unable to get " + path);
         }
     }
 
     @Nonnull
-    public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+    public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
         Object value = getValue(object, path);
-        if(value instanceof JsonObject) {
+        if (value instanceof JsonObject) {
             return (JsonObject) value;
-        }else {
+        } else {
             throw new ParsingException("Unable to get " + path);
         }
     }
 
     @Nonnull
-    public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
+    public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
         Object value = getValue(object, path);
-        if(value instanceof JsonArray) {
+        if (value instanceof JsonArray) {
             return (JsonArray) value;
-        }else {
+        } else {
             throw new ParsingException("Unable to get " + path);
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -185,5 +185,4 @@ public class Utils {
         }
         return uri.getProtocol() + "://" + uri.getAuthority();
     }
-    
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java

@@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
         extractor = (PeertubeCommentsExtractor) PeerTube
-                .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
+                .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
     }
 
     @Test
@@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
     @Test
     public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
         boolean result = false;
-        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
+        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
         assertTrue("Comments".equals(commentsInfo.getName()));
         result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java

@@ -11,6 +11,8 @@ import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
 
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -29,16 +31,28 @@ import org.schabi.newpipe.extractor.stream.StreamType;
  */
 public class PeertubeStreamExtractorDefaultTest {
     private static PeertubeStreamExtractor extractor;
+    private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
+    private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
         // setting instance might break test when running in parallel
-        PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
+        PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
-        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
+        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
         extractor.fetchPage();
     }
 
+    @Test
+    public void testGetUploadDate() throws ParsingException, ParseException {
+        final Calendar instance = Calendar.getInstance();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
+        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
+
+    }
+
     @Test
     public void testGetInvalidTimeStamp() throws ParsingException {
         assertTrue(extractor.getTimeStamp() + "",
@@ -47,22 +61,37 @@ public class PeertubeStreamExtractorDefaultTest {
 
     @Test
     public void testGetTitle() throws ParsingException {
-        assertEquals(extractor.getName(), "Power Corrupts the Best");
+        assertEquals("What is PeerTube?", extractor.getName());
     }
 
     @Test
-    public void testGetDescription() throws ParsingException {
+    public void testGetLargeDescription() throws ParsingException {
-        assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
+        assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetEmptyDescription() throws Exception {
+        PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
+        extractorEmpty.fetchPage();
+        assertEquals("", extractorEmpty.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetSmallDescription() throws Exception {
+        PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
+        PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
+        extractorSmall.fetchPage();
+        assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
     }
 
     @Test
     public void testGetUploaderName() throws ParsingException {
-        assertEquals(extractor.getUploaderName(), "Rowsedower");
+        assertEquals("Framasoft", extractor.getUploaderName());
     }
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(extractor.getLength(), 269);
+        assertEquals(113, extractor.getLength());
     }
 
     @Test
@@ -71,18 +100,10 @@ public class PeertubeStreamExtractorDefaultTest {
                 extractor.getViewCount() > 10);
     }
 
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
-        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-
-    }
-
     @Test
     public void testGetUploaderUrl() throws ParsingException {
         assertIsSecureUrl(extractor.getUploaderUrl());
-        assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
+        assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
     }
 
     @Test
@@ -115,11 +136,31 @@ public class PeertubeStreamExtractorDefaultTest {
 
     @Test
     public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
+        assertFalse(extractor.getSubtitlesDefault().isEmpty());
     }
 
     @Test
     public void testGetSubtitlesList() throws IOException, ExtractionException {
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
+        assertFalse(extractor.getSubtitlesDefault().isEmpty());
+    }
+
+    @Test
+    public void testGetAgeLimit() throws ExtractionException, IOException {
+        assertEquals(0, extractor.getAgeLimit());
+        PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7");
+        ageLimit.fetchPage();
+        assertEquals(18, ageLimit.getAgeLimit());
+    }
+
+    @Test
+    public void testGetSupportInformation() throws ExtractionException, IOException {
+        PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
+        supportInfoExtractor.fetchPage();
+        assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
+    }
+
+    @Test
+    public void testGetLanguageInformation() throws ParsingException {
+        assertEquals(new Locale("en"), extractor.getLanguageInfo());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java

@@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest {
 
     @Test
     public void testGetDescription() throws ParsingException {
-        assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
+        assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
     @Test
     public void testGetDescription() throws ParsingException {
         assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().isEmpty());
+        assertFalse(extractor.getDescription().getContent().isEmpty());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest {
     @Test
     public void testGetDescription() throws ParsingException {
         assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().isEmpty());
+        assertFalse(extractor.getDescription().getContent().isEmpty());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest {
         @Test
         public void testGetDescription() throws ParsingException {
             assertNotNull(extractor.getDescription());
-            assertFalse(extractor.getDescription().isEmpty());
+            assertFalse(extractor.getDescription().getContent().isEmpty());
         }
 
         @Test
         public void testGetFullLinksInDescription() throws ParsingException {
-            assertTrue(extractor.getDescription().contains("http://adele.com"));
+            assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
-            assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi..."));
+            assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi..."));
         }
 
         @Test
@@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest {
         @Test
         public void testGetDescription() throws ParsingException {
             assertNotNull(extractor.getDescription());
-            assertFalse(extractor.getDescription().isEmpty());
+            assertFalse(extractor.getDescription().getContent().isEmpty());
         }
 
         @Test
         public void testGetFullLinksInDescription() throws ParsingException {
-            assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
-            assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
-            assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
+            assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
 
-            assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub..."));
-            assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8..."));
-            assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/..."));
         }
     }
 
@@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest {
         @Test
         public void testGetDescription() throws ParsingException {
             assertNotNull(extractor.getDescription());
-            assertFalse(extractor.getDescription().isEmpty());
+            assertFalse(extractor.getDescription().getContent().isEmpty());
         }
 
         @Test
         public void testGetFullLinksInDescription() throws ParsingException {
-            assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
-            assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
-            assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
-            assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
 
-            assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
-            assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
-            assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
-            assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
+            assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest {
     @Test
     public void testGetDescription() throws ParsingException {
         assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().isEmpty());
+        assertFalse(extractor.getDescription().getContent().isEmpty());
     }
 
     @Test
     public void testGetFullLinksInDescription() throws ParsingException {
-        assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
+        assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/"));
-        assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
+        assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba..."));
     }
 
     @Test