added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails) test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted) metadata: got host, privacy (public, private, unlisted), licence, language, tags
This commit is contained in:
bopol
parent
2ee558fbe7
commit
a281519e77
|
|
@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
|
|||
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
|
||||
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
|
||||
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
|
||||
|
||||
|
||||
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
|
||||
|
||||
## Supported sites
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.stream.*;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
|
|
@ -225,4 +226,35 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
public String getOriginalUrl() throws ParsingException {
|
||||
return data.getString("frontend_link");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrivacy() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCategory() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLicence() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getStreamInfoLanguage() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() throws ParsingException {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import java.util.List;
|
|||
|
||||
import org.jsoup.helper.StringUtil;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.ServiceList;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
|
|
@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.StreamType;
|
|||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
|
|
@ -66,16 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
return baseUrl + JsonUtils.getString(json, "thumbnailPath");
|
||||
return baseUrl + JsonUtils.getString(json, "previewPath");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() throws ParsingException {
|
||||
String description = "";
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
try {
|
||||
return JsonUtils.getString(json, "description");
|
||||
}catch(ParsingException e) {
|
||||
return "No description";
|
||||
Response response = dl.get(getUrl() + "/description");
|
||||
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
|
||||
description = JsonUtils.getString(jsonObject, "description");
|
||||
} catch (ReCaptchaException | IOException | JsonParserException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
if (description.equals("")) {
|
||||
//if the request above failed
|
||||
description = JsonUtils.getString(json, "description");
|
||||
}
|
||||
return description;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -224,8 +232,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
|
||||
return collector;
|
||||
}
|
||||
|
||||
private List<String> getTags(){
|
||||
|
||||
@Override
|
||||
public List<String> getTags(){
|
||||
try {
|
||||
return (List) JsonUtils.getArray(json, "tags");
|
||||
} catch (Exception e) {
|
||||
|
|
@ -339,4 +348,29 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
return baseUrl + "/videos/watch/" + getId();
|
||||
}
|
||||
|
||||
//TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return JsonUtils.getString(json, "account.host");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrivacy() throws ParsingException {
|
||||
return JsonUtils.getString(json, "privacy.label");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCategory() throws ParsingException {
|
||||
return JsonUtils.getString(json, "category.label");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLicence() throws ParsingException {
|
||||
return JsonUtils.getString(json, "licence.label");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getStreamInfoLanguage() throws ParsingException {
|
||||
return JsonUtils.getString(json, "language.label");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -254,4 +254,35 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
public String getErrorMessage() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrivacy() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCategory() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLicence() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getStreamInfoLanguage() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() throws ParsingException {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1113,4 +1113,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
throw new ExtractionException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrivacy() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCategory() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLicence() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getStreamInfoLanguage() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() throws ParsingException {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
|
@ -349,4 +350,60 @@ public abstract class StreamExtractor extends Extractor {
|
|||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The host of the stream (Eg. peertube.cpy.re).
|
||||
* If the privacy is not available, or if the service doesn't use
|
||||
* a federated system, but a centralised system,
|
||||
* you can simply return an empty string.
|
||||
* @return the host of the stream or an empty String.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract String getHost() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The privacy of the stream (Eg. Public, Private, Unlisted…).
|
||||
* If the privacy is not available you can simply return an empty string.
|
||||
* @return the privacy of the stream or an empty String.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract String getPrivacy() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The name of the category of the stream.
|
||||
* If the category is not available you can simply return an empty string.
|
||||
* @return the category of the stream or an empty String.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract String getCategory() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The name of the licence of the stream.
|
||||
* If the licence is not available you can simply return an empty string.
|
||||
* @return the licence of the stream or an empty String.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract String getLicence() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The language of the stream.
|
||||
* If the language is not available you can simply return an empty string.
|
||||
* @return the licence of the stream or an empty String.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract String getStreamInfoLanguage() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The list of tags of the stream.
|
||||
* If the tag list is not available you can simply return an empty list.
|
||||
* @return the list of tags of the stream or an empty list.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract List<String> getTags() throws ParsingException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -270,6 +270,38 @@ public class StreamInfo extends Info {
|
|||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
//additional info
|
||||
try {
|
||||
streamInfo.setHost(extractor.getHost());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setPrivacy(extractor.getPrivacy());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setCategory(extractor.getCategory());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setLicence(extractor.getLicence());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setLanguage(extractor.getStreamInfoLanguage());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setTags(extractor.getTags());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
|
||||
|
||||
return streamInfo;
|
||||
|
|
@ -308,6 +340,13 @@ public class StreamInfo extends Info {
|
|||
private long startPosition = 0;
|
||||
private List<SubtitlesStream> subtitles = new ArrayList<>();
|
||||
|
||||
private String host = "";
|
||||
private String privacy = "";
|
||||
private String category = "";
|
||||
private String licence = "";
|
||||
private String language = "";
|
||||
private List<String> tags = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* Get the stream type
|
||||
*
|
||||
|
|
@ -533,4 +572,53 @@ public class StreamInfo extends Info {
|
|||
this.subtitles = subtitles;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return this.host;
|
||||
}
|
||||
|
||||
public void setHost(String str) {
|
||||
this.host = str;
|
||||
}
|
||||
|
||||
public String getPrivacy() {
|
||||
return this.privacy;
|
||||
}
|
||||
|
||||
public void setPrivacy(String str) {
|
||||
this.privacy = str;
|
||||
}
|
||||
|
||||
public String getCategory() {
|
||||
return this.category;
|
||||
}
|
||||
|
||||
public void setCategory(String cat) {
|
||||
this.category = cat;
|
||||
}
|
||||
|
||||
public String getLicence() {
|
||||
return this.licence;
|
||||
}
|
||||
|
||||
public void setLicence(String str) {
|
||||
this.licence = str;
|
||||
}
|
||||
|
||||
public String getLanguage() {
|
||||
return this.language;
|
||||
}
|
||||
|
||||
public void setLanguage(String lang) {
|
||||
this.language = lang;
|
||||
}
|
||||
|
||||
public List<String> getTags() {
|
||||
return this.tags;
|
||||
}
|
||||
|
||||
public void setTags(List<String> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -185,5 +185,4 @@ public class Utils {
|
|||
}
|
||||
return uri.getProtocol() + "://" + uri.getAuthority();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
|
|||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = (PeertubeCommentsExtractor) PeerTube
|
||||
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
|
||||
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
|
|||
@Test
|
||||
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
|
||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
|
||||
assertTrue("Comments".equals(commentsInfo.getName()));
|
||||
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
|
||||
|
||||
|
|
|
|||
|
|
@ -29,13 +29,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
|
|||
*/
|
||||
public class PeertubeStreamExtractorDefaultTest {
|
||||
private static PeertubeStreamExtractor extractor;
|
||||
private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
// setting instance might break test when running in parallel
|
||||
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
|
||||
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
|
||||
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
|
||||
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
|
|
@ -47,22 +48,22 @@ public class PeertubeStreamExtractorDefaultTest {
|
|||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertEquals(extractor.getName(), "Power Corrupts the Best");
|
||||
assertEquals("What is PeerTube?", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
|
||||
assertEquals(expectedDescription, extractor.getDescription());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertEquals(extractor.getUploaderName(), "Rowsedower");
|
||||
assertEquals("Framasoft", extractor.getUploaderName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertEquals(extractor.getLength(), 269);
|
||||
assertEquals(113, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -74,7 +75,7 @@ public class PeertubeStreamExtractorDefaultTest {
|
|||
@Test
|
||||
public void testGetUploadDate() throws ParsingException, ParseException {
|
||||
final Calendar instance = Calendar.getInstance();
|
||||
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
|
||||
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z"));
|
||||
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
|
||||
|
||||
}
|
||||
|
|
@ -82,7 +83,7 @@ public class PeertubeStreamExtractorDefaultTest {
|
|||
@Test
|
||||
public void testGetUploaderUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getUploaderUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
|
||||
assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -115,11 +116,11 @@ public class PeertubeStreamExtractorDefaultTest {
|
|||
|
||||
@Test
|
||||
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitlesDefault().isEmpty());
|
||||
assertFalse(extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesList() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitlesDefault().isEmpty());
|
||||
assertFalse(extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue