be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into soundcloudComments

This commit is contained in:
bopol 2020-05-05 14:49:21 +02:00
parent 2564bcf399 4086715a68
commit bc13e0c616
49 changed files with 880 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -3,8 +3,7 @@ jdk:
- openjdk8
script:
- ./gradlew check
- ./gradlew aggregatedJavadocs
- ./gradlew check aggregatedJavadocs
deploy:
provider: pages

3
build.gradle
View File

@ -5,11 +5,12 @@ allprojects {
sourceCompatibility = 1.7
targetCompatibility = 1.7
version 'v0.19.0'
version 'v0.19.4'
group 'com.github.TeamNewPipe'
repositories {
jcenter()
maven { url "https://jitpack.io" }
}
}

4
extractor/build.gradle
View File

@ -1,11 +1,11 @@
dependencies {
implementation project(':timeago-parser')
implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
testImplementation 'junit:junit:4.12'
}
}

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -6,7 +6,12 @@ import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
@ -21,19 +26,17 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class MediaCCCService extends StreamingService {
public MediaCCCService(int id) {
public MediaCCCService(final int id) {
super(id, "MediaCCC", asList(AUDIO, VIDEO));
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
return new MediaCCCSearchExtractor(this, query);
}
@ -58,17 +61,17 @@ public class MediaCCCService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new MediaCCCStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return null;
}
@ -85,9 +88,9 @@ public class MediaCCCService extends StreamingService {
try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String kioskId) throws ExtractionException, IOException {
public KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url, final String kioskId)
throws ExtractionException {
return new MediaCCCConferenceKiosk(MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId);
}
@ -111,8 +114,7 @@ public class MediaCCCService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) {
return null;
}

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -14,45 +15,46 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.Medi
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
import javax.annotation.Nonnull;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler) {
public MediaCCCConferenceExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getAvatarUrl() throws ParsingException {
public String getAvatarUrl() {
return conferenceData.getString("logo_url");
}
@Override
public String getBannerUrl() throws ParsingException {
public String getBannerUrl() {
return conferenceData.getString("logo_url");
}
@Override
public String getFeedUrl() throws ParsingException {
public String getFeedUrl() {
return null;
}
@Override
public long getSubscriberCount() throws ParsingException {
public long getSubscriberCount() {
return -1;
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return null;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
@ -62,17 +64,18 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) {
return null;
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
} catch (JsonParserException jpe) {
@ -88,7 +91,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
public String getOriginalUrl() {
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
}
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
@ -14,22 +15,22 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
import javax.annotation.Nonnull;
public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
private JsonObject doc;
public MediaCCCConferenceKiosk(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId) {
public MediaCCCConferenceKiosk(final StreamingService streamingService,
final ListLinkHandler linkHandler,
final String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Nonnull
@Override
public InfoItemsPage<ChannelInfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<ChannelInfoItem> getInitialPage() {
JsonArray conferences = doc.getArray("conferences");
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId());
for (int i = 0; i < conferences.size(); i++) {
@ -40,18 +41,20 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<ChannelInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<ChannelInfoItem> getPage(final String pageUrl) {
return InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()).responseBody();
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization())
.responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java
View File

@ -7,11 +7,10 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
public class MediaCCCParsingHelper {
private MediaCCCParsingHelper() {
}
public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -4,31 +4,34 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
public class MediaCCCSearchExtractor extends SearchExtractor {
private JsonObject doc;
private MediaCCCConferenceKiosk conferenceKiosk;
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
public MediaCCCSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
@ -40,13 +43,13 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
@Override
public String getSearchSuggestion() throws ParsingException {
public String getSearchSuggestion() {
return null;
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getInitialPage() {
final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId());
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
@ -70,17 +73,18 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final String pageUrl) {
return InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
if (getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty()) {
@ -95,44 +99,45 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty())
|| getLinkHandler().getContentFilters().isEmpty()) {
conferenceKiosk.fetchPage();
}
}
private void searchConferences(String searchString,
List<ChannelInfoItem> channelItems,
InfoItemsSearchCollector collector) {
private void searchConferences(final String searchString,
final List<ChannelInfoItem> channelItems,
final InfoItemsSearchCollector collector) {
for (final ChannelInfoItem item : channelItems) {
if (item.getName().toUpperCase().contains(
searchString.toUpperCase())) {
collector.commit(new ChannelInfoItemExtractor() {
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return item.getDescription();
}
@Override
public long getSubscriberCount() throws ParsingException {
public long getSubscriberCount() {
return item.getSubscriberCount();
}
@Override
public long getStreamCount() throws ParsingException {
public long getStreamCount() {
return item.getStreamCount();
}
@Override
public String getName() throws ParsingException {
public String getName() {
return item.getName();
}
@Override
public String getUrl() throws ParsingException {
public String getUrl() {
return item.getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return item.getThumbnailUrl();
}
});

87
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -11,27 +12,34 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor {
import javax.annotation.Nonnull;
public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
private JsonObject conferenceData;
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler) {
public MediaCCCStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getTextualUploadDate() throws ParsingException {
public String getTextualUploadDate() {
return data.getString("release_date");
}
@ -43,79 +51,79 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return data.getString("thumb_url");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
public Description getDescription() {
return new Description(data.getString("description"), Description.PLAIN_TEXT);
}
@Override
public int getAgeLimit() throws ParsingException {
public int getAgeLimit() {
return 0;
}
@Override
public long getLength() throws ParsingException {
public long getLength() {
return data.getInt("length");
}
@Override
public long getTimeStamp() throws ParsingException {
public long getTimeStamp() {
return 0;
}
@Override
public long getViewCount() throws ParsingException {
public long getViewCount() {
return data.getInt("view_count");
}
@Override
public long getLikeCount() throws ParsingException {
public long getLikeCount() {
return -1;
}
@Override
public long getDislikeCount() throws ParsingException {
public long getDislikeCount() {
return -1;
}
@Nonnull
@Override
public String getUploaderUrl() throws ParsingException {
public String getUploaderUrl() {
return data.getString("conference_url");
}
@Nonnull
@Override
public String getUploaderName() throws ParsingException {
public String getUploaderName() {
return data.getString("conference_url")
.replace("https://api.media.ccc.de/public/conferences/", "");
.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
}
@Nonnull
@Override
public String getUploaderAvatarUrl() throws ParsingException {
public String getUploaderAvatarUrl() {
return conferenceData.getString("logo_url");
}
@Nonnull
@Override
public String getDashMpdUrl() throws ParsingException {
public String getDashMpdUrl() {
return "";
}
@Nonnull
@Override
public String getHlsUrl() throws ParsingException {
public String getHlsUrl() {
return "";
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
public List<AudioStream> getAudioStreams() throws ExtractionException {
final JsonArray recordings = data.getArray("recordings");
final List<AudioStream> audioStreams = new ArrayList<>();
for (int i = 0; i < recordings.size(); i++) {
@ -134,14 +142,15 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
throw new ExtractionException("Unknown media format: " + mimeType);
}
audioStreams.add(new AudioStream(recording.getString("recording_url"), mediaFormat, -1));
audioStreams.add(new AudioStream(recording.getString("recording_url"),
mediaFormat, -1));
}
}
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
public List<VideoStream> getVideoStreams() throws ExtractionException {
final JsonArray recordings = data.getArray("recordings");
final List<VideoStream> videoStreams = new ArrayList<>();
for (int i = 0; i < recordings.size(); i++) {
@ -167,34 +176,34 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
public List<VideoStream> getVideoOnlyStreams() {
return null;
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
public List<SubtitlesStream> getSubtitlesDefault() {
return Collections.emptyList();
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
return Collections.emptyList();
}
@Override
public StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public StreamInfoItem getNextStream() throws IOException, ExtractionException {
public StreamInfoItem getNextStream() {
return null;
}
@Override
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
public StreamInfoItemsCollector getRelatedStreams() {
return new StreamInfoItemsCollector(getServiceId());
}
@ -204,14 +213,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
data = JsonParser.object().from(
downloader.get(getLinkHandler().getUrl()).responseBody());
conferenceData = JsonParser.object()
.from(downloader.get(getUploaderUrl()).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe);
throw new ExtractionException("Could not parse json returned by url: "
+ getLinkHandler().getUrl(), jpe);
}
}
@ -223,44 +234,44 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
public String getOriginalUrl() {
return data.getString("frontend_link");
}
@Override
public String getHost() throws ParsingException {
public String getHost() {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
public String getPrivacy() {
return "";
}
@Override
public String getCategory() throws ParsingException {
public String getCategory() {
return "";
}
@Override
public String getLicence() throws ParsingException {
public String getLicence() {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
public Locale getLanguageInfo() {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
public List<String> getTags() {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
public String getSupportInfo() {
return "";
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java
View File

@ -1,21 +0,0 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class MediaCCCSuggestionExtractor extends SuggestionExtractor {
public MediaCCCSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
return new ArrayList<>(0);
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java
View File

@ -5,25 +5,24 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor {
private JsonObject conference;
JsonObject conference;
public MediaCCCConferenceInfoItemExtractor(JsonObject conference) {
public MediaCCCConferenceInfoItemExtractor(final JsonObject conference) {
this.conference = conference;
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
return "";
}
@Override
public long getSubscriberCount() throws ParsingException {
public long getSubscriberCount() {
return -1;
}
@Override
public long getStreamCount() throws ParsingException {
public long getStreamCount() {
return -1;
}
@ -38,7 +37,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return conference.getString("logo_url");
}
}

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java
View File

@ -10,47 +10,46 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor {
private JsonObject event;
JsonObject event;
public MediaCCCStreamInfoItemExtractor(JsonObject event) {
public MediaCCCStreamInfoItemExtractor(final JsonObject event) {
this.event = event;
}
@Override
public StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() throws ParsingException {
public boolean isAd() {
return false;
}
@Override
public long getDuration() throws ParsingException {
public long getDuration() {
return event.getInt("length");
}
@Override
public long getViewCount() throws ParsingException {
public long getViewCount() {
return event.getInt("view_count");
}
@Override
public String getUploaderName() throws ParsingException {
public String getUploaderName() {
return event.getString("conference_url")
.replace("https://api.media.ccc.de/public/conferences/", "");
.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
}
@Override
public String getUploaderUrl() throws ParsingException {
public String getUploaderUrl() {
return event.getString("conference_url");
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
public String getTextualUploadDate() {
return event.getString("release_date");
}
@ -67,12 +66,12 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
@Override
public String getUrl() throws ParsingException {
return "https://api.media.ccc.de/public/events/" +
event.getString("guid");
return "https://media.ccc.de/public/events/"
+ event.getString("guid");
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return event.getString("thumb_url");
}
}

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java
View File

@ -7,16 +7,17 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
return "https://api.media.ccc.de/public/conferences/" + id;
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException {
return "https://media.ccc.de/public/conferences/" + id;
}
@Override
public String getId(String url) throws ParsingException {
if (url.startsWith("https://api.media.ccc.de/public/conferences/")) {
return url.replace("https://api.media.ccc.de/public/conferences/", "");
public String getId(final String url) throws ParsingException {
if (url.startsWith("https://media.ccc.de/public/conferences/")
|| url.startsWith("https://api.media.ccc.de/public/conferences/")) {
return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
} else if (url.startsWith("https://media.ccc.de/c/")) {
return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
} else if (url.startsWith("https://media.ccc.de/b/")) {
@ -26,7 +27,7 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
}
@Override
public boolean onAcceptUrl(String url) throws ParsingException {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java
View File

@ -7,18 +7,20 @@ import java.util.List;
public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
return "conferences";
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
return "https://api.media.ccc.de/public/conferences";
public String getUrl(final String id, final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return "https://media.ccc.de/public/conferences";
}
@Override
public boolean onAcceptUrl(String url) throws ParsingException {
public boolean onAcceptUrl(final String url) {
return url.equals("https://media.ccc.de/b/conferences")
|| url.equals("https://media.ccc.de/public/conferences")
|| url.equals("https://api.media.ccc.de/public/conferences");
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -8,7 +8,6 @@ import java.net.URLEncoder;
import java.util.List;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
public static final String EVENTS = "events";
@ -28,11 +27,13 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
}
@Override
public String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String query, final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
return "https://api.media.ccc.de/public/events/search?q=" + URLEncoder.encode(querry, "UTF-8");
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with querry: " + querry, e);
throw new ParsingException("Could not create search string with querry: " + query, e);
}
}
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
@ -9,11 +8,15 @@ import java.net.MalformedURLException;
import java.net.URL;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
@Override
public String getId(String urlString) throws ParsingException {
if (urlString.startsWith("https://api.media.ccc.de/public/events/") &&
!urlString.contains("?q=")) {
public String getId(final String urlString) throws ParsingException {
if (urlString.startsWith("https://media.ccc.de/public/events/")
&& !urlString.contains("?q=")) {
return urlString.substring(35); //remove …/public/events part
}
if (urlString.startsWith("https://api.media.ccc.de/public/events/")
&& !urlString.contains("?q=")) {
return urlString.substring(39); //remove api…/public/events part
}
@ -38,12 +41,12 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(String id) throws ParsingException {
return "https://api.media.ccc.de/public/events/" + id;
public String getUrl(final String id) throws ParsingException {
return "https://media.ccc.de/public/events/" + id;
}
@Override
public boolean onAcceptUrl(String url) throws ParsingException {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -77,7 +77,12 @@ public class PeertubeService extends StreamingService {
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelExtractor(this, linkHandler);
if (linkHandler.getUrl().contains("/video-channels/")) {
return new PeertubeChannelExtractor(this, linkHandler);
} else {
return new PeertubeAccountExtractor(this, linkHandler);
}
}
@Override

187
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java Normal file
View File

@ -0,0 +1,187 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
public class PeertubeAccountExtractor extends ChannelExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getAvatarUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getBannerUrl() throws ParsingException {
return null;
}
@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
}
@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
}
@Override
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
} catch (ParsingException e) {
return "No description";
}
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
PeertubeParsingHelper.validate(json);
Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if (nextStart >= total) {
return "";
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
}
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract PeerTube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id");
}
@Override
@ -181,7 +181,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/accounts/" + getId();
return baseUrl + "/" + getId();
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -97,7 +97,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -128,7 +128,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(json, "account.name");
String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override
@ -167,7 +167,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
JsonArray streams = json.getArray("files");
for (Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View File

@ -51,7 +51,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java
View File

@ -10,8 +10,8 @@ import java.util.List;
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "/accounts/([^/?&#]*)";
private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/";
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
private static final String API_ENDPOINT = "/api/v1/";
public static PeertubeChannelLinkHandlerFactory getInstance() {
return instance;
@ -19,7 +19,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
return Parser.matchGroup(ID_PATTERN, url, 0);
}
@Override
@ -31,11 +31,17 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
return baseUrl + ACCOUNTS_ENDPOINT + id;
if (id.matches(ID_PATTERN)) {
return baseUrl + API_ENDPOINT + id;
} else {
// This is needed for compatibility with older versions were we didn't support video channels yet
return baseUrl + API_ENDPOINT + "accounts/" + id;
}
}
@Override
public boolean onAcceptUrl(String url) {
return url.contains("/accounts/");
return url.contains("/accounts/") || url.contains("/video-channels/");
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -31,6 +31,7 @@ import java.util.*;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
@ -261,17 +262,17 @@ public class SoundcloudParsingHelper {
@Nonnull
public static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", "");
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
@Nonnull
public static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
public static String getUploaderName(JsonObject object) {
return object.getObject("user").getString("username", "");
return object.getObject("user").getString("username", EMPTY_STRING);
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -17,6 +17,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@ -63,10 +65,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() {
return user.getObject("visuals", new JsonObject())
.getArray("visuals", new JsonArray())
.getObject(0, new JsonObject())
.getString("visual_url");
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
}
@Override
@ -81,7 +80,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getDescription() {
return user.getString("description", "");
return user.getString("description", EMPTY_STRING);
}
@Nonnull

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
@ -24,7 +25,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", "");
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
}
@ -41,6 +42,6 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getDescription() {
return itemObject.getString("description", "");
return itemObject.getString("description", EMPTY_STRING);
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
@ -31,7 +32,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
@ -45,7 +46,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
@ -53,8 +54,8 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
}
// Then look for track creator avatar url
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
final JsonObject creator = trackObject.getObject(USER_KEY);
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
@ -63,7 +64,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -23,6 +23,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class SoundcloudSearchExtractor extends SearchExtractor {
@ -84,7 +85,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", "");
String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -35,6 +35,8 @@ import java.util.Locale;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -46,7 +48,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
String policy = track.getString("policy", "");
String policy = track.getString("policy", EMPTY_STRING);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
@ -79,9 +81,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() {
String artworkUrl = track.getString("artwork_url", "");
String artworkUrl = track.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = track.getObject("user").getString("avatar_url", "");
artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
@ -63,7 +64,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public String getThumbnailUrl() {
String artworkUrl = itemObject.getString("artwork_url", "");
String artworkUrl = itemObject.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = itemObject.getObject("user").getString("avatar_url");
}

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -28,6 +28,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@ -334,7 +335,7 @@ public class YoutubeParsingHelper {
}
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
@ -354,7 +355,7 @@ public class YoutubeParsingHelper {
} else if (internUrl.startsWith("http")) {
return internUrl;
}
} else if (navigationEndpoint.getObject("browseEndpoint") != null) {
} else if (navigationEndpoint.has("browseEndpoint")) {
final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
final String browseId = browseEndpoint.getString("browseId");
@ -369,7 +370,7 @@ public class YoutubeParsingHelper {
}
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")");
} else if (navigationEndpoint.getObject("watchEndpoint") != null) {
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
@ -377,20 +378,30 @@ public class YoutubeParsingHelper {
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
} else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) {
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
}
return null;
}
/**
* Get the text from a JSON object that has either a simpleText or a runs array.
* @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null}
*/
public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException {
if (textObject == null || textObject.isEmpty()) return null;
if (textObject.has("simpleText")) return textObject.getString("simpleText");
if (textObject.getArray("runs").isEmpty()) return null;
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
@ -484,12 +495,12 @@ public class YoutubeParsingHelper {
* @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected
*/
public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
public static void defaultAlertsCheck(final JsonObject initialData) throws ParsingException {
final JsonArray alerts = initialData.getArray("alerts");
if (alerts != null && !alerts.isEmpty()) {
if (!alerts.isEmpty()) {
final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
final String alertText = alertRenderer.getObject("text").getString("simpleText");
final String alertType = alertRenderer.getString("type");
final String alertText = getTextFromObject(alertRenderer.getObject("text"));
final String alertType = alertRenderer.getString("type", EMPTY_STRING);
if (alertType.equalsIgnoreCase("ERROR")) {
throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
}

73
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -10,8 +10,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
@ -20,7 +20,7 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -49,7 +49,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it.
*<p>
* <p>
* "Movies & Shows":
* <pre>
* UCuJcl0Ju-gPDoksRjK1ya-w ┐
@ -72,22 +72,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
while (level < 3) {
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT)
.getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY)
.getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT)
.getObject("endpoint", EMPTY_OBJECT);
final JsonObject endpoint = jsonResponse.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint
.getObject("commandMetadata", EMPTY_OBJECT)
.getObject("webCommandMetadata", EMPTY_OBJECT)
final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("webPageType", EMPTY_STRING);
final String browseId = endpoint
.getObject("browseEndpoint", EMPTY_OBJECT)
.getString("browseId", EMPTY_STRING);
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
@ -131,10 +125,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
final String channelId = initialData
.getObject("header", EMPTY_OBJECT)
.getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT)
.getString("channelId", EMPTY_STRING);
final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("channelId", EMPTY_STRING);
if (!channelId.isEmpty()) {
return channelId;
@ -170,11 +162,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
String url = null;
try {
url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
.getArray("thumbnails").getObject(0).getString("url");
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null;
}
@ -196,19 +186,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) {
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer");
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
try {
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText")));
} catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
// If there's no subscribe button, the channel has the subscriber count disabled
if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
return -1;
} else {
if (c4TabbedHeaderRenderer.has("subscribeButton")) {
return 0;
} else {
return -1;
}
}
}
@ -260,7 +250,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) return "";
if (continuations == null || continuations.isEmpty()) {
return "";
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
@ -277,7 +269,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
if (((JsonObject) video).has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
@Override
@ -302,8 +294,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
JsonObject videoTab = null;
for (Object tab : tabs) {
if (((JsonObject) tab).getObject("tabRenderer") != null) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
@ -314,13 +306,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ContentNotSupportedException("This channel has no Videos tab");
}
try {
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
} catch (Exception ignored) {}
final String messageRendererText = getTextFromObject(videoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
}
this.videoTab = videoTab;
return videoTab;

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -70,14 +70,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getSubscriberCount() throws ParsingException {
try {
final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText");
if (subscriberCountObject == null) {
if (!channelInfoItem.has("subscriberCountText")) {
// Subscription count is not available for this channel item.
return -1;
}
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject));
return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText")));
} catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e);
}
@ -86,14 +84,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getStreamCount() throws ParsingException {
try {
final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText");
if (videoCountObject == null) {
if (!channelInfoItem.has("videoCountText")) {
// Video count is not available, channel probably has no public uploads.
return -1;
}
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject)));
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
channelInfoItem.getObject("videoCountText"))));
} catch (Exception e) {
throw new ParsingException("Could not get stream count", e);
}
@ -102,14 +99,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getDescription() throws ParsingException {
try {
final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet");
if (descriptionObject == null) {
if (!channelInfoItem.has("descriptionSnippet")) {
// Channel have no description.
return null;
}
return getTextFromObject(descriptionObject);
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) {
throw new ParsingException("Could not get description", e);
}

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -28,15 +28,13 @@ import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
@ -128,14 +126,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
if (itemSectionRenderer == null) {
return "";
}
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer");
if (didYouMeanRenderer == null) {
final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("didYouMeanRenderer");
if (!didYouMeanRenderer.has("correctedQuery")) {
return "";
}
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
@ -149,7 +143,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
if (((JsonObject) content).has("musicShelfRenderer")) {
collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents"));
}
}
@ -162,7 +156,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
if (((JsonObject) content).has("musicShelfRenderer")) {
return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations"));
}
}
@ -224,10 +218,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
throw new ParsingException("Could not parse JSON", e);
}
if (ajaxJson.getObject("continuationContents") == null) {
return InfoItemsPage.emptyPage();
}
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation");
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
@ -240,7 +230,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer");
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null);
if (info != null) {
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
@ -290,27 +280,26 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items");
for (Object item : items) {
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer");
if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) {
if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) {
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint"));
}
}
return null;
} else {
final JsonObject navigationEndpoint = info.getArray("flexColumns")
final JsonObject navigationEndpointHolder = info.getArray("flexColumns")
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint");
.getObject("text").getArray("runs").getObject(0);
if (navigationEndpoint == null) {
return null;
}
if (!navigationEndpointHolder.has("navigationEndpoint")) return null;
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get uploader url");
throw new ParsingException("Could not get uploader URL");
}
}
@ -480,7 +469,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -47,23 +47,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject getUploaderInfo() throws ParsingException {
JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
try {
JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner").getObject("videoOwnerRenderer");
if (uploaderInfo != null) {
return uploaderInfo;
}
} catch (Exception ignored) {}
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
// we might want to create a loop here instead of using duplicated code
try {
JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner").getObject("videoOwnerRenderer");
if (uploaderInfo != null) {
return uploaderInfo;
}
} catch (Exception e) {
throw new ParsingException("Could not get uploader info", e);
videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
throw new ParsingException("Could not get uploader info");
}
@ -89,33 +82,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getName() throws ParsingException {
try {
String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null) return name;
} catch (Exception ignored) {}
try {
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
} catch (Exception e) {
throw new ParsingException("Could not get playlist name", e);
}
String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}
@Override
public String getThumbnailUrl() throws ParsingException {
String url = null;
String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
try {
url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null || url.isEmpty()) {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
if (url == null) {
try {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get playlist thumbnail");
if (url == null || url.isEmpty()) throw new ParsingException("Could not get playlist thumbnail");
}
return fixThumbnailUrl(url);
@ -123,8 +105,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getBannerUrl() {
return ""; // Banner can't be handled by frontend right now.
// Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement it in the frontend.
return "";
}
@Override
@ -199,7 +182,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}
@ -216,7 +199,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
if (((JsonObject) video).getObject("playlistVideoRenderer") != null) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
@Override
public long getViewCount() {

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -69,7 +69,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (showingResultsForRenderer == null) {
if (!showingResultsForRenderer.has("correctedQuery")) {
return "";
}
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
@ -119,21 +119,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
if (((JsonObject) item).has("backgroundPromoRenderer")) {
throw new NothingFoundException(getTextFromObject(((JsonObject) item)
.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (((JsonObject) item).getObject("videoRenderer") != null) {
} else if (((JsonObject) item).has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
} else if (((JsonObject) item).getObject("channelRenderer") != null) {
} else if (((JsonObject) item).has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer")));
} else if (((JsonObject) item).getObject("playlistRenderer") != null) {
} else if (((JsonObject) item).has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer")));
}
}
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}

151
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -33,7 +33,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -53,10 +52,8 @@ import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 06.08.15.
@ -117,18 +114,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getName() throws ParsingException {
assertPageFetched();
String title = null;
String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
try {
title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
} catch (Exception ignored) {}
if (title == null || title.isEmpty()) {
title = playerResponse.getObject("videoDetails").getString("title");
if (title == null) {
try {
title = playerResponse.getObject("videoDetails").getString("title");
} catch (Exception ignored) {}
if (title == null) throw new ParsingException("Could not get name");
if (title == null || title.isEmpty()) throw new ParsingException("Could not get name");
}
return title;
@ -140,35 +131,31 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return null;
}
try {
JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) {
return micro.getString("uploadDate");
}
if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) {
return micro.getString("publishDate");
}
} catch (Exception ignored) {}
JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) {
return micro.getString("uploadDate");
}
if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) {
return micro.getString("publishDate");
}
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
}
try {
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
}
} catch (Exception ignored) {}
try {
// TODO this parses English formatted dates only, we need a better approach to parse the textual date
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
@ -180,7 +167,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
if (textualUploadDate == null || textualUploadDate.isEmpty()) {
return null;
}
@ -208,17 +195,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public Description getDescription() throws ParsingException {
assertPageFetched();
// description with more info on links
try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
return new Description(description, Description.HTML);
} catch (Exception ignored) { }
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
if (description != null && !description.isEmpty()) return new Description(description, Description.HTML);
// raw non-html description
try {
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
} catch (Exception ignored) {
throw new ParsingException("Could not get description");
}
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
}
@Override
@ -264,19 +245,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() throws ParsingException {
assertPageFetched();
String views = null;
try {
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
} catch (Exception ignored) {}
if (views == null) {
try {
views = playerResponse.getObject("videoDetails").getString("viewCount");
} catch (Exception ignored) {}
if (views == null || views.isEmpty()) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (views == null) throw new ParsingException("Could not get view count");
if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count");
}
if (views.toLowerCase().contains("no views")) return 0;
@ -334,16 +309,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
assertPageFetched();
try {
String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (uploaderUrl != null) return uploaderUrl;
} catch (Exception ignored) {}
try {
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null)
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
} catch (Exception ignored) {}
if (uploaderUrl != null && !uploaderUrl.isEmpty()) return uploaderUrl;
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null && !uploaderId.isEmpty())
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
throw new ParsingException("Could not get uploader url");
}
@ -351,19 +326,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getUploaderName() throws ParsingException {
assertPageFetched();
String uploaderName = null;
try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (Exception ignored) {}
if (uploaderName == null) {
try {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
} catch (Exception ignored) {}
if (uploaderName == null || uploaderName.isEmpty()) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (uploaderName == null) throw new ParsingException("Could not get uploader name");
if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name");
}
return uploaderName;
@ -392,7 +361,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs != null && playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd", "");
dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING);
} else {
return "";
}
@ -561,9 +530,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object ul : results) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer");
if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
if (((JsonObject) ul).has("compactVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser));
}
}
return collector;
} catch (Exception e) {
@ -612,7 +581,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String playerUrl;
if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos
initialData = initialAjaxJson.getObject(2).getObject("response");
ageLimit = 18;
@ -631,7 +600,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerResponse = getPlayerResponse();
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT);
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
final String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw a ContentNotAvailableException with the reason.
if (status != null && !status.toLowerCase().equals("ok")) {
@ -808,10 +777,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
captions = playerResponse.getObject("captions");
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// todo: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
// This check is necessary since there may be cases where subtitles metadata do not contain caption track info
// e.g. https://www.youtube.com/watch?v=-Vpwatutnko
@ -876,7 +845,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject videoPrimaryInfoRenderer = null;
for (Object content : contents) {
if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer");
break;
}
@ -898,7 +867,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject videoSecondaryInfoRenderer = null;
for (Object content : contents) {
if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer");
break;
}

119
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -17,6 +17,7 @@ import java.util.Calendar;
import java.util.Date;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -37,7 +38,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
*/
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private JsonObject videoInfo;
private final TimeAgoParser timeAgoParser;
private StreamType cachedStreamType;
@ -59,23 +59,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return cachedStreamType;
}
try {
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
}
} catch (Exception ignored) {}
try {
final String style = videoInfo.getArray("thumbnailOverlays").getObject(0)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style");
if (style.equalsIgnoreCase("LIVE")) {
final JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("LIVE NOW")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
} catch (Exception ignored) {}
}
final String style = videoInfo.getArray("thumbnailOverlays").getObject(0)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING);
if (style.equalsIgnoreCase("LIVE")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
return cachedStreamType = StreamType.VIDEO_STREAM;
}
@ -108,23 +103,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return -1;
}
String duration = null;
String duration = getTextFromObject(videoInfo.getObject("lengthText"));
try {
duration = getTextFromObject(videoInfo.getObject("lengthText"));
} catch (Exception ignored) {}
if (duration == null) {
try {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
if (duration == null || duration.isEmpty()) {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
} catch (Exception ignored) {}
}
if (duration == null) throw new ParsingException("Could not get duration");
if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration");
}
return YoutubeParsingHelper.parseDurationString(duration);
@ -132,23 +121,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderName() throws ParsingException {
String name = null;
String name = getTextFromObject(videoInfo.getObject("longBylineText"));
try {
name = getTextFromObject(videoInfo.getObject("longBylineText"));
} catch (Exception ignored) {}
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("ownerText"));
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("ownerText"));
} catch (Exception ignored) {}
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
} catch (Exception ignored) {}
if (name == null) throw new ParsingException("Could not get uploader name");
if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name");
}
}
@ -157,26 +138,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
String url = null;
String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
if (url == null || url.isEmpty()) {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
if (url == null || url.isEmpty()) {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get uploader url");
if (url == null || url.isEmpty()) throw new ParsingException("Could not get uploader url");
}
}
@ -195,12 +168,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
}
try {
return getTextFromObject(videoInfo.getObject("publishedTimeText"));
} catch (Exception e) {
// upload date is not always available, e.g. in playlists
return null;
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText;
return null;
}
@Nullable
@ -228,17 +199,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public long getViewCount() throws ParsingException {
try {
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
if (videoInfo.has("topStandaloneBadge") || isPremium()) {
return -1;
}
final JsonObject viewCountObject = videoInfo.getObject("viewCountText");
if (viewCountObject == null) {
if (!videoInfo.has("viewCountText")) {
// This object is null when a video has its views hidden.
return -1;
}
final String viewCount = getTextFromObject(viewCountObject);
final String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
if (viewCount.toLowerCase().contains("no views")) {
return 0;
@ -266,14 +236,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
private boolean isPremium() {
try {
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) {
return true;
}
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) {
return true;
}
} catch (Exception ignored) {
}
return false;
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -59,7 +59,7 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
// trim JSONP part "JP(...)"
response = response.substring(3, response.length() - 1);
try {
JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray());
JsonArray collection = JsonParser.array().from(response).getArray(1);
for (Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) continue;
String suggestionStr = ((JsonArray) suggestion).getString(0);

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -72,12 +72,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull
@Override
public String getName() throws ParsingException {
String name;
try {
name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
} catch (Exception e) {
throw new ParsingException("Could not get Trending name", e);
}
String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
if (name != null && !name.isEmpty()) {
return name;
}
@ -97,14 +92,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");
if (expandedShelfContentsRenderer != null) {
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View File

@ -14,7 +14,6 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
* Test {@link MediaCCCConferenceExtractor}
*/
public class MediaCCCConferenceExtractorTest {
public static class FrOSCon2017 {
private static MediaCCCConferenceExtractor extractor;
@ -32,7 +31,7 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetUrl() throws Exception {
assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl());
assertEquals("https://media.ccc.de/public/conferences/froscon2017", extractor.getUrl());
}
@Test
@ -68,7 +67,7 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetUrl() throws Exception {
assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl());
assertEquals("https://media.ccc.de/public/conferences/oscal19", extractor.getUrl());
}
@Test

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java
View File

@ -15,14 +15,14 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
* Test {@link MediaCCCStreamExtractor}
*/
public class MediaCCCOggTest {
// test against https://api.media.ccc.de/public/events/1317
// test against https://media.ccc.de/public/events/1317
private static StreamExtractor extractor;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317");
extractor = MediaCCC.getStreamExtractor("https://media.ccc.de/public/events/1317");
extractor.fetchPage();
}

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.UtilsTest;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@ -25,7 +24,6 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
* Test {@link MediaCCCStreamExtractor}
*/
public class MediaCCCStreamExtractorTest {
public static class Gpn18Tmux {
private static MediaCCCStreamExtractor extractor;
@ -55,7 +53,7 @@ public class MediaCCCStreamExtractorTest {
@Test
public void testUrl() throws Exception {
assertIsSecureUrl(extractor.getUrl());
assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
assertEquals("https://media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
}
@Test
@ -78,7 +76,7 @@ public class MediaCCCStreamExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
}
@Test
@ -141,7 +139,7 @@ public class MediaCCCStreamExtractorTest {
@Test
public void testUrl() throws Exception {
assertIsSecureUrl(extractor.getUrl());
assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
assertEquals("https://media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
}
@Test
@ -164,7 +162,7 @@ public class MediaCCCStreamExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
assertEquals("https://media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
}
@Test
@ -203,4 +201,4 @@ public class MediaCCCStreamExtractorTest {
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
}
}
}

13
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java
View File

@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaC
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
public class MediaCCCSearchExtractorTest {
public static class All extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "kde";
@ -32,8 +31,8 @@ public class MediaCCCSearchExtractorTest {
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@ -55,8 +54,8 @@ public class MediaCCCSearchExtractorTest {
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@ -79,8 +78,8 @@ public class MediaCCCSearchExtractorTest {
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }

205
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java Normal file
View File

@ -0,0 +1,205 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link PeertubeAccountExtractor}
*/
public class PeertubeAccountExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
private static PeertubeAccountExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/kde", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
}
}
public static class Booteille implements BaseChannelExtractorTest {
private static PeertubeAccountExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/booteille", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
}
}
}

33
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
@ -20,7 +19,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link PeertubeChannelExtractor}
*/
public class PeertubeChannelExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
public static class DanDAugeTutoriels implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -29,7 +28,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
.getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
extractor.fetchPage();
}
@ -44,22 +43,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
assertEquals("Dan d'Auge tutoriels", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("kde", extractor.getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -98,16 +97,16 @@ public class PeertubeChannelExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl());
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4);
}
}
public static class Booteille implements BaseChannelExtractorTest {
public static class Divers implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -116,7 +115,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
.getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
extractor.fetchPage();
}
@ -141,22 +140,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
assertEquals("Divers", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("booteille", extractor.getId());
assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -195,12 +194,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl());
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
}
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java
View File

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChanne
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
/**
* Test for {@link PeertubeChannelLinkHandlerFactory}
@ -19,6 +20,7 @@ public class PeertubeChannelLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
linkHandler = PeertubeChannelLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@ -26,11 +28,20 @@ public class PeertubeChannelLinkHandlerFactoryTest {
@Test
public void acceptUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
}
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
}
@Test
public void getUrlFromId() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl());
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -17,6 +17,7 @@ import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.TimeZone;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
@ -84,7 +85,9 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000"));
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
instance.setTime(sdf.parse("2016/07/31 18:18:07 +0000"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}

4
timeago-parser/build.gradle
View File

@ -1,6 +1,6 @@
dependencies {
testImplementation 'junit:junit:4.12'
implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
}
}