From a281519e771d139e7d6cd1b7f64dece92cbb5124 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 19 Jan 2020 12:45:52 +0100 Subject: [PATCH 01/18] added metadata, fix descriptions, fix thumbnail, update tests thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails) test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted) metadata: got host, privacy (public, private, unlisted), licence, language, tags --- README.md | 2 +- .../extractors/MediaCCCStreamExtractor.java | 32 +++++++ .../extractors/PeertubeStreamExtractor.java | 50 +++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 31 +++++++ .../extractors/YoutubeStreamExtractor.java | 31 +++++++ .../extractor/stream/StreamExtractor.java | 57 ++++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 88 +++++++++++++++++++ .../schabi/newpipe/extractor/utils/Utils.java | 1 - .../PeertubeCommentsExtractorTest.java | 4 +- .../PeertubeStreamExtractorDefaultTest.java | 21 ++--- 10 files changed, 295 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8c9c8cf2..66515055d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) - + > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. ## Supported sites diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index a1dfd2454..84fb76859 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -225,4 +226,35 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public String getOriginalUrl() throws ParsingException { return data.getString("frontend_link"); } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d8d0de005..0bead848c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -9,7 +9,7 @@ import java.util.List; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -66,16 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "previewPath"); } @Override public String getDescription() throws ParsingException { + String description = ""; + Downloader dl = NewPipe.getDownloader(); try { - return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { - return "No description"; + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + description = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); } + if (description.equals("")) { + //if the request above failed + description = JsonUtils.getString(json, "description"); + } + return description; } @Override @@ -224,8 +232,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } - - private List getTags(){ + + @Override + public List getTags(){ try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -339,4 +348,29 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } + //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it + @Override + public String getHost() throws ParsingException { + return JsonUtils.getString(json, "account.host"); + } + + @Override + public String getPrivacy() throws ParsingException { + return JsonUtils.getString(json, "privacy.label"); + } + + @Override + public String getCategory() throws ParsingException { + return JsonUtils.getString(json, "category.label"); + } + + @Override + public String getLicence() throws ParsingException { + return JsonUtils.getString(json, "licence.label"); + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return JsonUtils.getString(json, "language.label"); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 14f7023f4..19ab987cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -254,4 +254,35 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public String getErrorMessage() { return null; } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ede..0e380fbce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1113,4 +1113,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ExtractionException(e); } } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index d697dff2c..ce18dfa3b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -349,4 +350,60 @@ public abstract class StreamExtractor extends Extractor { return 0; } } + + /** + * The host of the stream (Eg. peertube.cpy.re). + * If the privacy is not available, or if the service doesn't use + * a federated system, but a centralised system, + * you can simply return an empty string. + * @return the host of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getHost() throws ParsingException; + + /** + * The privacy of the stream (Eg. Public, Private, Unlisted…). + * If the privacy is not available you can simply return an empty string. + * @return the privacy of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getPrivacy() throws ParsingException; + + /** + * The name of the category of the stream. + * If the category is not available you can simply return an empty string. + * @return the category of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getCategory() throws ParsingException; + + /** + * The name of the licence of the stream. + * If the licence is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getLicence() throws ParsingException; + + /** + * The language of the stream. + * If the language is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getStreamInfoLanguage() throws ParsingException; + + /** + * The list of tags of the stream. + * If the tag list is not available you can simply return an empty list. + * @return the list of tags of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getTags() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 34ea703c2..111f387d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -270,6 +270,38 @@ public class StreamInfo extends Info { streamInfo.addError(e); } + //additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (Exception e) { + streamInfo.addError(e); + } + streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -308,6 +340,13 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles = new ArrayList<>(); + private String host = ""; + private String privacy = ""; + private String category = ""; + private String licence = ""; + private String language = ""; + private List tags = new ArrayList<>(); + /** * Get the stream type * @@ -533,4 +572,53 @@ public class StreamInfo extends Info { this.subtitles = subtitles; } + public String getHost() { + return this.host; + } + + public void setHost(String str) { + this.host = str; + } + + public String getPrivacy() { + return this.privacy; + } + + public void setPrivacy(String str) { + this.privacy = str; + } + + public String getCategory() { + return this.category; + } + + public void setCategory(String cat) { + this.category = cat; + } + + public String getLicence() { + return this.licence; + } + + public void setLicence(String str) { + this.licence = str; + } + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String lang) { + this.language = lang; + } + + public List getTags() { + return this.tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fd06d40f5..3489b6d60 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -185,5 +185,4 @@ public class Utils { } return uri.getProtocol() + "://" + uri.getAuthority(); } - } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index fef1c3ee5..e0a1d1893 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } @Test @@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertTrue("Comments".equals(commentsInfo.getName())); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 75a692d42..b2175e851 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,13 +29,14 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; + private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); extractor.fetchPage(); } @@ -47,22 +48,22 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Power Corrupts the Best"); + assertEquals("What is PeerTube?", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + assertEquals(expectedDescription, extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "Rowsedower"); + assertEquals("Framasoft", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 269); + assertEquals(113, extractor.getLength()); } @Test @@ -74,7 +75,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } @@ -82,7 +83,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); } @Test @@ -115,11 +116,11 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } } From b382416372add1c9420cf1657f67c9cae825751f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 14:36:12 +0100 Subject: [PATCH 02/18] changed the way to extract peertube description if the description length is above 254, and ends with ..., it means the description we got from the first request is shortened. why above 254: because in fact, shortened descriptions length are between 255 : https://framatube.org/videos/watch/24efbe1b-73c0-4d72-a3ff-77c8b32d3fcd https://framatube.org/videos/watch/1ca99f40-fb5b-4fa4-abe5-4d54325df7fc and 269: https://framatube.org/videos/watch/4d661d5f-a884-4510-bca8-15cb19aa3fe5 also fixed a typo in StreamExtractor.java --- .../extractors/PeertubeStreamExtractor.java | 25 +++++++++++-------- .../extractor/stream/StreamExtractor.java | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0bead848c..ac3d6ecbe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -70,20 +70,23 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getDescription() throws ParsingException { - String description = ""; - Downloader dl = NewPipe.getDownloader(); + String desc; try { - Response response = dl.get(getUrl() + "/description"); - JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - description = JsonUtils.getString(jsonObject, "description"); - } catch (ReCaptchaException | IOException | JsonParserException e) { - e.printStackTrace(); + desc = JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; } - if (description.equals("")) { - //if the request above failed - description = JsonUtils.getString(json, "description"); + if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + Downloader dl = NewPipe.getDownloader(); + try { + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + desc = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); + } } - return description; + return desc; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index ce18dfa3b..a638c7b6d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -353,7 +353,7 @@ public abstract class StreamExtractor extends Extractor { /** * The host of the stream (Eg. peertube.cpy.re). - * If the privacy is not available, or if the service doesn't use + * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. * @return the host of the stream or an empty String. From ad7f97ae836f89a0f555e1062e5cc3b5211b1dc3 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 15:14:23 +0100 Subject: [PATCH 03/18] fix PeerTube description and add more description tests actually, the max description length is 250 after request with our extractor. during my tests, I made API requests with Firefox, copy/pasted into echo "insert description" | wc, and it was giving a wrong length, maybe due to the escapers, I have no idea anyway, it's now fixed --- .../extractors/PeertubeStreamExtractor.java | 3 ++- .../PeertubeStreamExtractorDefaultTest.java | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3d6ecbe..c922b2fef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index b2175e851..2a4378a20 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,7 +29,8 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; - private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; @BeforeClass public static void setUp() throws Exception { @@ -52,8 +53,23 @@ public class PeertubeStreamExtractorDefaultTest { } @Test - public void testGetDescription() throws ParsingException { - assertEquals(expectedDescription, extractor.getDescription()); + public void testGetLargeDescription() throws ParsingException { + assertEquals(expectedLargeDescription, extractor.getDescription()); + } + + @Test + public void testGetEmptyDescription() throws Exception { + PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("No description", extractorEmpty.getDescription()); + } + + @Test + public void testGetSmallDescription() throws Exception { + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals(expectedSmallDescription, extractorSmall.getDescription()); } @Test From f403490bd9ada2b19176ffc694d5b5ad405975b6 Mon Sep 17 00:00:00 2001 From: B0pol Date: Mon, 20 Jan 2020 21:28:26 +0100 Subject: [PATCH 04/18] Refactoring Co-Authored-By: Tobias Groza --- .../services/peertube/extractors/PeertubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c922b2fef..0de72a940 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { From 1a15c0e750dc95fa0a92cc1c988e781e66631fa7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Thu, 23 Jan 2020 04:42:54 +0100 Subject: [PATCH 05/18] agelimit now returns 18 if the video is marked as nsfw, 0 otherwise + created getBoolean method in JsonUtils.java --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../org/schabi/newpipe/extractor/utils/JsonUtils.java | 10 ++++++++++ .../peertube/PeertubeStreamExtractorDefaultTest.java | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0de72a940..b841d9fd9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -92,7 +92,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - return NO_AGE_LIMIT; + boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); + if (isNSFW) { + return 18; + } else { + return NO_AGE_LIMIT; + } } @Override @@ -352,7 +357,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } - //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it @Override public String getHost() throws ParsingException { return JsonUtils.getString(json, "account.host"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e5d7bb62f..ebd251d7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -37,6 +37,16 @@ public class JsonUtils { throw new ParsingException("Unable to get " + path); } } + + @Nonnull + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Boolean) { + return (Boolean) value; + }else { + throw new ParsingException("Unable to get " + path); + } + } @Nonnull public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 2a4378a20..4166b192f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -139,4 +139,12 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetSubtitlesList() throws IOException, ExtractionException { assertFalse(extractor.getSubtitlesDefault().isEmpty()); } + + @Test + public void testGetAgeLimit() throws ExtractionException, IOException { + assertEquals(0, extractor.getAgeLimit()); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7"); + ageLimit.fetchPage(); + assertEquals(18, ageLimit.getAgeLimit()); + } } From 74439f692a234d89b5556e1949192f3f9681f840 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:19:22 +0100 Subject: [PATCH 06/18] add extraction for support info and rename getLanguageInfo function --- .../extractors/MediaCCCStreamExtractor.java | 8 +++++++- .../extractors/PeertubeStreamExtractor.java | 12 ++++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 8 +++++++- .../extractors/YoutubeStreamExtractor.java | 8 +++++++- .../extractor/stream/StreamExtractor.java | 14 +++++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 18 +++++++++++++++--- .../PeertubeStreamExtractorDefaultTest.java | 7 +++++++ 7 files changed, 66 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 84fb76859..9168d6e61 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -248,7 +248,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -257,4 +257,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index b841d9fd9..4dadae778 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -35,6 +35,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import javax.annotation.Nonnull; + public class PeertubeStreamExtractor extends StreamExtractor { @@ -250,7 +252,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return Collections.emptyList(); } } - + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return JsonUtils.getString(json, "support"); + } + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); @@ -378,7 +386,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return JsonUtils.getString(json, "language.label"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 19ab987cd..c5d34bd01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -276,7 +276,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -285,4 +285,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ea9a84d46..74a44a665 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1156,7 +1156,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -1165,4 +1165,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a638c7b6d..31a5a583b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -396,7 +396,7 @@ public abstract class StreamExtractor extends Extractor { * @throws ParsingException */ @Nonnull - public abstract String getStreamInfoLanguage() throws ParsingException; + public abstract String getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. @@ -406,4 +406,16 @@ public abstract class StreamExtractor extends Extractor { */ @Nonnull public abstract List getTags() throws ParsingException; + + /** + * The support information of the stream. + * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37 + * (support button). + * If the support information are not available, + * you can simply return an empty String. + * @return the support information of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getSupportInfo() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 111f387d7..e32a133d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -292,7 +292,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); } catch (Exception e) { streamInfo.addError(e); } @@ -301,6 +301,11 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); @@ -345,6 +350,7 @@ public class StreamInfo extends Info { private String category = ""; private String licence = ""; private String language = ""; + private String support = ""; private List tags = new ArrayList<>(); /** @@ -604,11 +610,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguage() { + public String getLanguageInfo() { return this.language; } - public void setLanguage(String lang) { + public void setLanguageInfo(String lang) { this.language = lang; } @@ -620,5 +626,11 @@ public class StreamInfo extends Info { this.tags = tags; } + public void setSupportInfo(String support) { + this.support = support; + } + public String getSupportInfo() { + return this.support; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 4166b192f..829678526 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -147,4 +147,11 @@ public class PeertubeStreamExtractorDefaultTest { ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } } From 20da4750f8f779e7fd8efaf9471b890cd47c75a7 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:37:14 +0100 Subject: [PATCH 07/18] empty support returns "", same for empty description --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 4dadae778..a57720e34 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { desc = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return "No description"; + return ""; } if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description @@ -256,7 +256,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getSupportInfo() throws ParsingException { - return JsonUtils.getString(json, "support"); + try { + return JsonUtils.getString(json, "support"); + } catch (ParsingException e) { + return ""; + } } private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 829678526..962735062 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -61,7 +61,7 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("No description", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription()); } @Test From a691d6d0fc4949ab1392082865416e83a04af341 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:02:40 +0100 Subject: [PATCH 08/18] fix upload date: there was a one hour delay on peertube.co.uk: https://i.imgur.com/8Pai1rb.png on newpipe, before this commit: https://i.imgur.com/NIRbs4Z.png on newpipe, with this commit: https://i.ibb.co/mhKKtRD/Screenshot-20200123-185422-New-Pipe-Debug.png --- .../extractor/services/peertube/PeertubeParsingHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f518faf9a..2483dd949 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,6 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.concurrent.TimeUnit; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -27,6 +28,7 @@ public class PeertubeParsingHelper { Date date; try { date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From c790261eed4ba148b603325075bcbbfbd3e9b4c9 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:08:41 +0100 Subject: [PATCH 09/18] update test --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 962735062..c6a164f74 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -91,7 +91,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } From 812c8e0ad25de2f1d3f5da16975cae1b145072f1 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 00:22:05 +0100 Subject: [PATCH 10/18] authorName in comments now follow PeerTube website MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea anonyme zirbeldrüse is his displayName, what was displayed in NewPipe. but on the website, it's shown 777@mastodon.xyz --- .../extractors/PeertubeCommentsInfoItemExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index edb59e1fe..0bcce9fc4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - }catch(Exception e) { + } catch(Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getString(item, "account.displayName"); + return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override From 341372c0d0dadcce1b88333d9c8afc58ec50fff7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 20:16:24 +0100 Subject: [PATCH 11/18] reindenting (ctrl alt l) on JsonUtils and PeertubeStreamExtractor --- .../extractors/PeertubeStreamExtractor.java | 75 ++++++++++--------- .../newpipe/extractor/utils/JsonUtils.java | 46 ++++++------ 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a57720e34..e6e32d44a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -39,16 +39,16 @@ import javax.annotation.Nonnull; public class PeertubeStreamExtractor extends StreamExtractor { - + + private final String baseUrl; private JsonObject json; private List subtitles = new ArrayList<>(); - private final String baseUrl; - + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(json, "publishedAt"); @@ -64,7 +64,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public String getThumbnailUrl() throws ParsingException { return baseUrl + JsonUtils.getString(json, "previewPath"); @@ -149,7 +149,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String value; try { value = JsonUtils.getString(json, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -176,8 +176,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { List videoStreams = new ArrayList<>(); try { JsonArray streams = json.getArray("files", new JsonArray()); - for(Object s: streams) { - if(!(s instanceof JsonObject)) continue; + for (Object s : streams) { + if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); @@ -211,8 +211,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { List filteredSubs = new ArrayList<>(); - for(SubtitlesStream sub: subtitles) { - if(sub.getFormat() == format) { + for (SubtitlesStream sub : subtitles) { + if (sub.getFormat() == format) { filteredSubs.add(sub); } } @@ -234,18 +234,18 @@ public class PeertubeStreamExtractor extends StreamExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); List tags = getTags(); String apiUrl = null; - if(!tags.isEmpty()) { + if (!tags.isEmpty()) { apiUrl = getRelatedStreamsUrl(tags); - - }else { + + } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } @Override - public List getTags(){ + public List getTags() { try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -267,7 +267,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); - for(String tag : tags) { + for (String tag : tags) { params.append("&tagsOneOf="); params.append(URLEncoder.encode(tag, "UTF-8")); } @@ -277,38 +277,38 @@ public class PeertubeStreamExtractor extends StreamExtractor { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for related videos", e); } } - - if(relatedVideosJson != null) { + + if (relatedVideosJson != null) { collectStreamsFrom(collector, relatedVideosJson); } } - + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract related videos", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams - if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } - + } - + @Override public String getErrorMessage() { @@ -318,12 +318,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + loadSubtitles(); } @@ -333,24 +333,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube stream data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + if (null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } - + private void loadSubtitles() { if (subtitles.isEmpty()) { try { - Response response = getDownloader().get(getUrl() + "/captions"); + Response response = getDownloader().get(getUrl() + "/captions"); JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for(Object c: captions) { - if(c instanceof JsonObject) { - JsonObject caption = (JsonObject)c; + for (Object c : captions) { + if (c instanceof JsonObject) { + JsonObject caption = (JsonObject) c; String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + if (fmt != null && languageCode != null) + subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); } } } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ebd251d7c..80e0dd58f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -16,71 +16,71 @@ public class JsonUtils { private JsonUtils() { } - + @Nonnull - public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); Object result = object.get(keys.get(keys.size() - 1)); - if(null == result) throw new ParsingException("Unable to get " + path); + if (null == result) throw new ParsingException("Unable to get " + path); return result; } - + @Nonnull - public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof String) { + if (value instanceof String) { return (String) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull - public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Boolean) { + if (value instanceof Boolean) { return (Boolean) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Number) { + if (value instanceof Number) { return (Number) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonObject) { + if (value instanceof JsonObject) { return (JsonObject) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonArray) { + if (value instanceof JsonArray) { return (JsonArray) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { - + List result = new ArrayList<>(); for (int i = 0; i < array.size(); i++) { JsonObject obj = array.getObject(i); From e392b6c68f247e954a2e4eac5c097a4d809d3e1f Mon Sep 17 00:00:00 2001 From: B0pol Date: Sat, 25 Jan 2020 13:16:42 +0100 Subject: [PATCH 12/18] getLanguageInfo returns Locale instead of String so that java can automatically translate with Locale.getDisplayLanguage(), instead of always having English name of the language --- .../extractors/MediaCCCStreamExtractor.java | 5 +++-- .../extractors/PeertubeStreamExtractor.java | 9 +++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 5 +++-- .../youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- .../newpipe/extractor/stream/StreamExtractor.java | 13 ++++++++----- .../schabi/newpipe/extractor/stream/StreamInfo.java | 7 ++++--- .../PeertubeStreamExtractorDefaultTest.java | 6 ++++++ 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 9168d6e61..07c36d127 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -18,6 +18,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public class MediaCCCStreamExtractor extends StreamExtractor { @@ -248,8 +249,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index e6e32d44a..80cae88a6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -6,6 +6,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; @@ -391,7 +392,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return JsonUtils.getString(json, "language.label"); + public Locale getLanguageInfo() throws ParsingException { + try { + return new Locale(JsonUtils.getString(json, "language.id")); + } catch (ParsingException e) { + return null; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c5d34bd01..5b1153d54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -20,6 +20,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -276,8 +277,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index dcf302cbd..10ffb1432 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1157,8 +1157,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 31a5a583b..652e0116a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -35,6 +35,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; /** * Scrapes information from a video/audio streaming service (eg, YouTube). @@ -390,13 +391,15 @@ public abstract class StreamExtractor extends Extractor { public abstract String getLicence() throws ParsingException; /** - * The language of the stream. - * If the language is not available you can simply return an empty string. - * @return the licence of the stream or an empty String. + * The locale language of the stream. + * If the language is not available you can simply return null. + * If the language is provided by a language code, you can return + * new Locale(language_code); + * @return the locale language of the stream or null. * @throws ParsingException */ - @Nonnull - public abstract String getLanguageInfo() throws ParsingException; + @Nullable + public abstract Locale getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index e32a133d0..64999ce79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /* * Created by Christian Schabesberger on 26.08.15. @@ -349,8 +350,8 @@ public class StreamInfo extends Info { private String privacy = ""; private String category = ""; private String licence = ""; - private String language = ""; private String support = ""; + private Locale language = null; private List tags = new ArrayList<>(); /** @@ -610,11 +611,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguageInfo() { + public Locale getLanguageInfo() { return this.language; } - public void setLanguageInfo(String lang) { + public void setLanguageInfo(Locale lang) { this.language = lang; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index c6a164f74..06a8352cc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Locale; import org.junit.BeforeClass; import org.junit.Ignore; @@ -154,4 +155,9 @@ public class PeertubeStreamExtractorDefaultTest { supportInfoExtractor.fetchPage(); assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); } + + @Test + public void testGetLanguageInformation() throws ParsingException { + assertEquals(new Locale("en"), extractor.getLanguageInfo()); + } } From 5756df8dc7e89b7383d1d1e07a91c30bdab6f868 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 22:24:28 +0100 Subject: [PATCH 13/18] Use GMT as base time (actually fix upload date) --- .../services/peertube/PeertubeParsingHelper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 2483dd949..c84e4e5b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,7 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import java.util.concurrent.TimeUnit; +import java.util.TimeZone; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -13,22 +13,23 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import com.grack.nanojson.JsonObject; public class PeertubeParsingHelper { - + private PeertubeParsingHelper() { } public static void validate(JsonObject json) throws ContentNotAvailableException { String error = json.getString("error"); - if(!StringUtil.isBlank(error)) { + if (!StringUtil.isBlank(error)) { throw new ContentNotAvailableException(error); } } - + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); - date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From 26c65b2948dcc58be4a004938f415bbd46380203 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 23:35:46 +0100 Subject: [PATCH 14/18] Create class Description --- .../extractors/MediaCCCStreamExtractor.java | 5 +- .../extractors/PeertubeStreamExtractor.java | 23 ++++------ .../soundcloud/SoundcloudStreamExtractor.java | 5 +- .../extractors/YoutubeStreamExtractor.java | 6 +-- .../newpipe/extractor/stream/Description.java | 46 +++++++++++++++++++ .../extractor/stream/StreamExtractor.java | 6 +-- .../newpipe/extractor/stream/StreamInfo.java | 6 +-- .../PeertubeStreamExtractorDefaultTest.java | 7 +-- ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- ...utubeStreamExtractorControversialTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 38 +++++++-------- .../YoutubeStreamExtractorLivestreamTest.java | 6 +-- 12 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 07c36d127..56036c2eb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,7 +12,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -49,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { - return data.getString("description"); + public Description getDescription() throws ParsingException { + return new Description(getServiceId(), data.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 80cae88a6..ac3267f14 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -21,14 +21,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import com.grack.nanojson.JsonArray; @@ -72,25 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getDescription() throws ParsingException { - String desc; + public Description getDescription() throws ParsingException { + String text; try { - desc = JsonUtils.getString(json, "description"); + text = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return ""; + return Description.emptyDescription; } - if (desc.length() == 250 && desc.substring(247).equals("...")) { + if (text.length() == 250 && text.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - desc = JsonUtils.getString(jsonObject, "description"); + text = JsonUtils.getString(jsonObject, "description"); } catch (ReCaptchaException | IOException | JsonParserException e) { e.printStackTrace(); } } - return desc; + return new Description(getServiceId(), text); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5b1153d54..c2a537a28 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -74,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return artworkUrlBetterResolution; } - @Nonnull @Override - public String getDescription() { - return track.getString("description"); + public Description getDescription() { + return new Description(getServiceId(), track.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f8523ea59..d8ded292d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { + public Description getDescription() throws ParsingException { assertPageFetched(); try { // first try to get html-formatted description - return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); + return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); } catch (Exception e) { try { // fallback to raw non-html description - return playerResponse.getObject("videoDetails").getString("shortDescription"); + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); } catch (Exception ignored) { throw new ParsingException("Could not get the description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java new file mode 100644 index 000000000..1ce5e948b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.stream; + +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class Description { + private String content; + private int type; + + public static final int HTML = 1; + public static final int MARKDOWN = 2; + public static final int PLAIN_TEXT = 3; + public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + + public Description(int serviceID, String content) { + if (serviceID == PeerTube.getServiceId()) { + this.type = MARKDOWN; + } else if (serviceID == YouTube.getServiceId()) { + this.type = HTML; + } else { + this.type = PLAIN_TEXT; + } + setContent(content); + } + + private void setContent(String content) { + if (content == null) { + this.content = ""; + } else { + this.content = content; + } + } + + public Description(String content, int type) { + this.type = type; + setContent(content); + } + + public String getContent() { + return content; + } + + public int getType() { + return type; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 652e0116a..fdc56972c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -84,12 +84,12 @@ public abstract class StreamExtractor extends Extractor { public abstract String getThumbnailUrl() throws ParsingException; /** - * This is the stream description. On YouTube this is the video description. You can return simple HTML here. - * @return The description of the stream/video. + * This is the stream description. + * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @Nonnull - public abstract String getDescription() throws ParsingException; + public abstract Description getDescription() throws ParsingException; /** * Get the age limit. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 64999ce79..580f9c213 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -319,7 +319,7 @@ public class StreamInfo extends Info { private DateWrapper uploadDate; private long duration = -1; private int ageLimit = -1; - private String description; + private Description description; private long viewCount = -1; private long likeCount = -1; @@ -417,11 +417,11 @@ public class StreamInfo extends Info { this.ageLimit = ageLimit; } - public String getDescription() { + public Description getDescription() { return description; } - public void setDescription(String description) { + public void setDescription(Description description) { this.description = description; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 06a8352cc..794a14a3b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -55,14 +55,14 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetLargeDescription() throws ParsingException { - assertEquals(expectedLargeDescription, extractor.getDescription()); + assertEquals(expectedLargeDescription, extractor.getDescription().getContent()); } @Test public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription().getContent()); } @Test @@ -70,7 +70,7 @@ public class PeertubeStreamExtractorDefaultTest { PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); extractorSmall.fetchPage(); - assertEquals(expectedSmallDescription, extractorSmall.getDescription()); + assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent()); } @Test @@ -89,6 +89,7 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } + @Ignore //fixme @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index b9da6d173..6afc7212a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index f0bd7715c..915f42ae1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 5f21df553..e2c0c7bbf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://adele.com")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); + assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); + assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); + assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } @@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index eb2541edf..b1d6f53a9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); + assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); } @Test From 70a40e7388f5bed7ca2019e0d4f82c18631f8902 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:28:27 +0100 Subject: [PATCH 15/18] Description: rm constructor by serviceId --- .../extractors/MediaCCCStreamExtractor.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../soundcloud/SoundcloudStreamExtractor.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 2 +- .../newpipe/extractor/stream/Description.java | 27 ++++--------------- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 56036c2eb..08f21ef73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -49,7 +49,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public Description getDescription() throws ParsingException { - return new Description(getServiceId(), data.getString("description")); + return new Description(data.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3267f14..a4dafbfc8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -83,7 +83,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { e.printStackTrace(); } } - return new Description(getServiceId(), text); + return new Description(text, Description.MARKDOWN); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c2a537a28..18fffa413 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -76,7 +76,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public Description getDescription() { - return new Description(getServiceId(), track.getString("description")); + return new Description(track.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d8ded292d..ea870d715 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -184,7 +184,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { // first try to get html-formatted description - return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); + return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML); } catch (Exception e) { try { // fallback to raw non-html description diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index 1ce5e948b..cde422bb2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,29 +1,17 @@ package org.schabi.newpipe.extractor.stream; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - public class Description { - private String content; - private int type; public static final int HTML = 1; public static final int MARKDOWN = 2; public static final int PLAIN_TEXT = 3; - public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + public static final Description emptyDescription = new Description("", PLAIN_TEXT); - public Description(int serviceID, String content) { - if (serviceID == PeerTube.getServiceId()) { - this.type = MARKDOWN; - } else if (serviceID == YouTube.getServiceId()) { - this.type = HTML; - } else { - this.type = PLAIN_TEXT; - } - setContent(content); - } + private String content; + private int type; - private void setContent(String content) { + public Description(String content, int type) { + this.type = type; if (content == null) { this.content = ""; } else { @@ -31,11 +19,6 @@ public class Description { } } - public Description(String content, int type) { - this.type = type; - setContent(content); - } - public String getContent() { return content; } From 0f8a7f9df2138d0e5697cc723655d8ff082699be Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:33:50 +0100 Subject: [PATCH 16/18] fix testGetUploadDate for PeerTubeStreamExtractor --- .../PeertubeStreamExtractorDefaultTest.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 794a14a3b..a81bf34cb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -12,6 +12,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Locale; +import java.util.TimeZone; import org.junit.BeforeClass; import org.junit.Ignore; @@ -42,6 +43,16 @@ public class PeertubeStreamExtractorDefaultTest { extractor.fetchPage(); } + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + + } + @Test public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(extractor.getTimeStamp() + "", @@ -89,15 +100,6 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } - @Ignore //fixme - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - - } - @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); From 11bcc78d9c8eb39e8d61a6f4bc4112025937f087 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 14:10:54 +0100 Subject: [PATCH 17/18] Description implements Serializable. fix NotSerializableException --- .../java/org/schabi/newpipe/extractor/stream/Description.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index cde422bb2..b19b8de36 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.stream; -public class Description { +import java.io.Serializable; + +public class Description implements Serializable { public static final int HTML = 1; public static final int MARKDOWN = 2; From 0e33249c85d2f23be9d8aa7fccd4bdc409a8f8ac Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 22:20:20 +0100 Subject: [PATCH 18/18] Fix SoundCloud description test --- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 8a3883fd6..0ffe06223 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); } @Test