Apply changes in all StreamExtractors except YouTube's one and fix extraction of PeerTube audio streams as video streams
Some code in these classes has been also refactored/improved/optimized. Also fix the extraction of PeerTube audio streams as video streams, which are now returned as audio streams.
This commit is contained in:
TiA4f8R
parent
d5f3637fc3
commit
881969f1da
|
|
@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
|
|||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
|
|
@ -30,9 +29,12 @@ import java.util.List;
|
|||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
||||
|
||||
private static final String OPUS_LO = "opus_lo";
|
||||
private static final String MP3_128 = "mp3-128";
|
||||
private JsonObject showInfo;
|
||||
|
||||
public BandcampRadioStreamExtractor(final StreamingService service,
|
||||
|
|
@ -78,11 +80,9 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
return Jsoup.parse(showInfo.getString("image_caption")).getElementsByTag("a").stream()
|
||||
.map(Element::text)
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ParsingException("Could not get uploader name"));
|
||||
public String getUploaderName() {
|
||||
return Jsoup.parse(showInfo.getString("image_caption"))
|
||||
.getElementsByTag("a").first().text();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
|
|
@ -116,23 +116,25 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
|||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
final ArrayList<AudioStream> list = new ArrayList<>();
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
final JsonObject streams = showInfo.getObject("audio_stream");
|
||||
|
||||
if (streams.has("opus-lo")) {
|
||||
list.add(new AudioStream(
|
||||
streams.getString("opus-lo"),
|
||||
MediaFormat.OPUS, 100
|
||||
));
|
||||
}
|
||||
if (streams.has("mp3-128")) {
|
||||
list.add(new AudioStream(
|
||||
streams.getString("mp3-128"),
|
||||
MediaFormat.MP3, 128
|
||||
));
|
||||
if (streams.has(MP3_128)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(MP3_128)
|
||||
.setContent(streams.getString(MP3_128), true)
|
||||
.setMediaFormat(MediaFormat.MP3)
|
||||
.setAverageBitrate(128)
|
||||
.build());
|
||||
} else if (streams.has(OPUS_LO)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(OPUS_LO)
|
||||
.setContent(streams.getString(OPUS_LO), true)
|
||||
.setMediaFormat(MediaFormat.OPUS)
|
||||
.setAverageBitrate(100).build());
|
||||
}
|
||||
|
||||
return list;
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -156,14 +158,14 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
|||
@Override
|
||||
public String getLicence() {
|
||||
// Contrary to other Bandcamp streams, radio streams don't have a license
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getCategory() {
|
||||
// Contrary to other Bandcamp streams, radio streams don't have categories
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
|
|
@ -10,7 +12,6 @@ import com.grack.nanojson.JsonParserException;
|
|||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
|
|
@ -27,16 +28,15 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
|
|||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class BandcampStreamExtractor extends StreamExtractor {
|
||||
|
||||
private JsonObject albumJson;
|
||||
private JsonObject current;
|
||||
private Document document;
|
||||
|
|
@ -88,7 +88,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
public String getUploaderUrl() throws ParsingException {
|
||||
final String[] parts = getUrl().split("/");
|
||||
// https: (/) (/) * .bandcamp.com (/) and leave out the rest
|
||||
return "https://" + parts[2] + "/";
|
||||
return HTTPS + parts[2] + "/";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -119,7 +119,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
if (albumJson.isNull("art_id")) {
|
||||
return Utils.EMPTY_STRING;
|
||||
return EMPTY_STRING;
|
||||
} else {
|
||||
return getImageUrl(albumJson.getLong("art_id"), true);
|
||||
}
|
||||
|
|
@ -139,24 +139,26 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
public Description getDescription() {
|
||||
final String s = Utils.nonEmptyAndNullJoin(
|
||||
"\n\n",
|
||||
new String[]{
|
||||
new String[] {
|
||||
current.getString("about"),
|
||||
current.getString("lyrics"),
|
||||
current.getString("credits")
|
||||
}
|
||||
);
|
||||
});
|
||||
return new Description(s, Description.PLAIN_TEXT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
|
||||
audioStreams.add(new AudioStream(
|
||||
albumJson.getArray("trackinfo").getObject(0)
|
||||
.getObject("file").getString("mp3-128"),
|
||||
MediaFormat.MP3, 128
|
||||
));
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId("mp3-128")
|
||||
.setContent(albumJson.getArray("trackinfo")
|
||||
.getObject(0)
|
||||
.getObject("file")
|
||||
.getString("mp3-128"), true)
|
||||
.setMediaFormat(MediaFormat.MP3)
|
||||
.setAverageBitrate(128)
|
||||
.build());
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
|
|
@ -184,11 +186,11 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public PlaylistInfoItemsCollector getRelatedItems() {
|
||||
final PlaylistInfoItemsCollector collector = new PlaylistInfoItemsCollector(getServiceId());
|
||||
final Elements recommendedAlbums = document.getElementsByClass("recommended-album");
|
||||
document.getElementsByClass("recommended-album")
|
||||
.stream()
|
||||
.map(BandcampRelatedPlaylistInfoItemExtractor::new)
|
||||
.forEach(collector::commit);
|
||||
|
||||
for (final Element album : recommendedAlbums) {
|
||||
collector.commit(new BandcampRelatedPlaylistInfoItemExtractor(album));
|
||||
}
|
||||
return collector;
|
||||
}
|
||||
|
||||
|
|
@ -200,15 +202,17 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
.flatMap(element -> element.getElementsByClass("tag").stream())
|
||||
.map(Element::text)
|
||||
.findFirst()
|
||||
.orElse("");
|
||||
.orElse(EMPTY_STRING);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getLicence() {
|
||||
/* Tests resulted in this mapping of ints to licence:
|
||||
/*
|
||||
Tests resulted in this mapping of ints to licence:
|
||||
https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview (screenshot from a Bandcamp artist's
|
||||
account) */
|
||||
account)
|
||||
*/
|
||||
|
||||
switch (current.getInt("license_type")) {
|
||||
case 1:
|
||||
|
|
@ -233,14 +237,9 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() {
|
||||
final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords");
|
||||
|
||||
final List<String> tags = new ArrayList<>();
|
||||
|
||||
for (final Element e : tagElements) {
|
||||
tags.add(e.text());
|
||||
}
|
||||
|
||||
return tags;
|
||||
return document.getElementsByAttributeValue("itemprop", "keywords")
|
||||
.stream()
|
||||
.map(Element::text)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
|
@ -17,11 +18,21 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
||||
private static final String STREAMS = "streams";
|
||||
private static final String URLS = "urls";
|
||||
private static final String URL = "url";
|
||||
|
||||
private JsonObject conference = null;
|
||||
private String group = "";
|
||||
private JsonObject room = null;
|
||||
|
|
@ -34,19 +45,22 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
final JsonArray doc =
|
||||
MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
|
||||
// find correct room
|
||||
final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
|
||||
getExtractorLocalization());
|
||||
// Find the correct room
|
||||
for (int c = 0; c < doc.size(); c++) {
|
||||
conference = doc.getObject(c);
|
||||
final JsonArray groups = conference.getArray("groups");
|
||||
final JsonObject conferenceObject = doc.getObject(c);
|
||||
final JsonArray groups = conferenceObject.getArray("groups");
|
||||
for (int g = 0; g < groups.size(); g++) {
|
||||
group = groups.getObject(g).getString("group");
|
||||
final String groupObject = groups.getObject(g).getString("group");
|
||||
final JsonArray rooms = groups.getObject(g).getArray("rooms");
|
||||
for (int r = 0; r < rooms.size(); r++) {
|
||||
room = rooms.getObject(r);
|
||||
if (getId().equals(
|
||||
conference.getString("slug") + "/" + room.getString("slug"))) {
|
||||
final JsonObject roomObject = rooms.getObject(r);
|
||||
if (getId().equals(conferenceObject.getString("slug") + "/"
|
||||
+ roomObject.getString("slug"))) {
|
||||
this.conference = conferenceObject;
|
||||
this.group = groupObject;
|
||||
this.room = roomObject;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -91,69 +105,136 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
|||
return conference.getString("conference");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL of the first DASH stream found.
|
||||
*
|
||||
* <p>
|
||||
* There can be several DASH streams, so the URL of the first found is returned by this method.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* You can find the other video DASH streams by using {@link #getVideoStreams()}
|
||||
* </p>
|
||||
*/
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getDashMpdUrl() throws ParsingException {
|
||||
|
||||
for (int s = 0; s < room.getArray(STREAMS).size(); s++) {
|
||||
final JsonObject stream = room.getArray(STREAMS).getObject(s);
|
||||
final JsonObject urls = stream.getObject(URLS);
|
||||
if (urls.has("dash")) {
|
||||
return urls.getObject("dash").getString(URL, EMPTY_STRING);
|
||||
}
|
||||
}
|
||||
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL of the first HLS stream found.
|
||||
*
|
||||
* <p>
|
||||
* There can be several HLS streams, so the URL of the first found is returned by this method.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* You can find the other video HLS streams by using {@link #getVideoStreams()}
|
||||
* </p>
|
||||
*/
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() {
|
||||
// TODO: There are multiple HLS streams.
|
||||
// Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams,
|
||||
// so the user can choose a resolution.
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("video")) {
|
||||
if (stream.has("hls")) {
|
||||
return stream.getObject("urls").getObject("hls").getString("url");
|
||||
}
|
||||
for (int s = 0; s < room.getArray(STREAMS).size(); s++) {
|
||||
final JsonObject stream = room.getArray(STREAMS).getObject(s);
|
||||
final JsonObject urls = stream.getObject(URLS);
|
||||
if (urls.has("hls")) {
|
||||
return urls.getObject("hls").getString(URL, EMPTY_STRING);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("audio")) {
|
||||
for (final String type : stream.getObject("urls").keySet()) {
|
||||
final JsonObject url = stream.getObject("urls").getObject(type);
|
||||
audioStreams.add(new AudioStream(url.getString("url"),
|
||||
MediaFormat.getFromSuffix(type), -1));
|
||||
}
|
||||
}
|
||||
}
|
||||
IntStream.range(0, room.getArray(STREAMS).size())
|
||||
.mapToObj(s -> room.getArray(STREAMS).getObject(s))
|
||||
.filter(streamJsonObject -> streamJsonObject.getString("type").equals("audio"))
|
||||
.forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet()
|
||||
.forEach(type -> {
|
||||
final JsonObject urlObject = streamJsonObject.getObject(URLS)
|
||||
.getObject(type);
|
||||
// The DASH manifest will be extracted with getDashMpdUrl
|
||||
if (!type.equals("dash")) {
|
||||
final AudioStream.Builder builder = new AudioStream.Builder()
|
||||
.setId(urlObject.getString("tech", ID_UNKNOWN))
|
||||
.setContent(urlObject.getString(URL), true)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE);
|
||||
if (type.equals("hls")) {
|
||||
// We don't know with the type string what media format will
|
||||
// have HLS streams.
|
||||
// However, the tech string may contain some information
|
||||
// about the media format used.
|
||||
builder.setDeliveryMethod(DeliveryMethod.HLS);
|
||||
} else {
|
||||
builder.setMediaFormat(MediaFormat.getFromSuffix(type));
|
||||
}
|
||||
|
||||
audioStreams.add(builder.build());
|
||||
}
|
||||
}));
|
||||
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("video")) {
|
||||
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
|
||||
+ stream.getArray("videoSize").getInt(1);
|
||||
for (final String type : stream.getObject("urls").keySet()) {
|
||||
if (!type.equals("hls")) {
|
||||
final JsonObject url = stream.getObject("urls").getObject(type);
|
||||
videoStreams.add(new VideoStream(
|
||||
url.getString("url"),
|
||||
MediaFormat.getFromSuffix(type),
|
||||
resolution));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
IntStream.range(0, room.getArray(STREAMS).size())
|
||||
.mapToObj(s -> room.getArray(STREAMS).getObject(s))
|
||||
.filter(stream -> stream.getString("type").equals("video"))
|
||||
.forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet()
|
||||
.forEach(type -> {
|
||||
final String resolution =
|
||||
streamJsonObject.getArray("videoSize").getInt(0)
|
||||
+ "x"
|
||||
+ streamJsonObject.getArray("videoSize").getInt(1);
|
||||
final JsonObject urlObject = streamJsonObject.getObject(URLS)
|
||||
.getObject(type);
|
||||
// The DASH manifest will be extracted with getDashMpdUrl
|
||||
if (!type.equals("dash")) {
|
||||
final VideoStream.Builder builder = new VideoStream.Builder()
|
||||
.setId(urlObject.getString("tech", ID_UNKNOWN))
|
||||
.setContent(urlObject.getString(URL), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(resolution);
|
||||
|
||||
if (type.equals("hls")) {
|
||||
// We don't know with the type string what media format will
|
||||
// have HLS streams.
|
||||
// However, the tech string may contain some information
|
||||
// about the media format used.
|
||||
builder.setDeliveryMethod(DeliveryMethod.HLS);
|
||||
} else {
|
||||
builder.setMediaFormat(MediaFormat.getFromSuffix(type));
|
||||
}
|
||||
|
||||
videoStreams.add(builder.build());
|
||||
}
|
||||
}));
|
||||
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
return null;
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
|
||||
return StreamType.LIVE_STREAM;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
|
|
@ -99,7 +102,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
final JsonObject recording = recordings.getObject(i);
|
||||
final String mimeType = recording.getString("mime_type");
|
||||
if (mimeType.startsWith("audio")) {
|
||||
//first we need to resolve the actual video data from CDN
|
||||
// First we need to resolve the actual video data from CDN
|
||||
final MediaFormat mediaFormat;
|
||||
if (mimeType.endsWith("opus")) {
|
||||
mediaFormat = MediaFormat.OPUS;
|
||||
|
|
@ -108,11 +111,18 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
} else if (mimeType.endsWith("ogg")) {
|
||||
mediaFormat = MediaFormat.OGG;
|
||||
} else {
|
||||
throw new ExtractionException("Unknown media format: " + mimeType);
|
||||
mediaFormat = null;
|
||||
}
|
||||
|
||||
audioStreams.add(new AudioStream(recording.getString("recording_url"),
|
||||
mediaFormat, -1));
|
||||
// Don't use the containsSimilarStream method because it will always return
|
||||
// false so if there are multiples audio streams available, only the first will
|
||||
// be extracted in this case.
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(recording.getString("filename", ID_UNKNOWN))
|
||||
.setContent(recording.getString("recording_url"), true)
|
||||
.setMediaFormat(mediaFormat)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
return audioStreams;
|
||||
|
|
@ -126,7 +136,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
final JsonObject recording = recordings.getObject(i);
|
||||
final String mimeType = recording.getString("mime_type");
|
||||
if (mimeType.startsWith("video")) {
|
||||
//first we need to resolve the actual video data from CDN
|
||||
// First we need to resolve the actual video data from CDN
|
||||
|
||||
final MediaFormat mediaFormat;
|
||||
if (mimeType.endsWith("webm")) {
|
||||
|
|
@ -134,13 +144,21 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
} else if (mimeType.endsWith("mp4")) {
|
||||
mediaFormat = MediaFormat.MPEG_4;
|
||||
} else {
|
||||
throw new ExtractionException("Unknown media format: " + mimeType);
|
||||
mediaFormat = null;
|
||||
}
|
||||
|
||||
videoStreams.add(new VideoStream(recording.getString("recording_url"),
|
||||
mediaFormat, recording.getInt("height") + "p"));
|
||||
// Don't use the containsSimilarStream method because it will remove the
|
||||
// extraction of some video versions (mostly languages)
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(recording.getString("filename", ID_UNKNOWN))
|
||||
.setContent(recording.getString("recording_url"), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setMediaFormat(mediaFormat)
|
||||
.setResolution(recording.getInt("height") + "p")
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
|
|
@ -163,7 +181,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
conferenceData = JsonParser.object()
|
||||
.from(downloader.get(data.getString("conference_url")).responseBody());
|
||||
} catch (final JsonParserException jpe) {
|
||||
throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
|
||||
throw new ExtractionException("Could not parse json returned by URL: " + videoUrl,
|
||||
jpe);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
|
|||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
|
|
@ -39,14 +40,30 @@ import java.util.ArrayList;
|
|||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
private static final String ACCOUNT_HOST = "account.host";
|
||||
private static final String ACCOUNT_NAME = "account.name";
|
||||
private static final String FILES = "files";
|
||||
private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl";
|
||||
private static final String FILE_URL = "fileUrl";
|
||||
private static final String PLAYLIST_URL = "playlistUrl";
|
||||
private static final String RESOLUTION_ID = "resolution.id";
|
||||
private static final String STREAMING_PLAYLISTS = "streamingPlaylists";
|
||||
|
||||
private final String baseUrl;
|
||||
private JsonObject json;
|
||||
|
||||
private final List<SubtitlesStream> subtitles = new ArrayList<>();
|
||||
private final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
private final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler)
|
||||
throws ParsingException {
|
||||
|
|
@ -85,9 +102,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
} catch (final ParsingException e) {
|
||||
return Description.EMPTY_DESCRIPTION;
|
||||
}
|
||||
|
||||
if (text.length() == 250 && text.substring(247).equals("...")) {
|
||||
//if description is shortened, get full description
|
||||
// If description is shortened, get full description
|
||||
final Downloader dl = NewPipe.getDownloader();
|
||||
try {
|
||||
final Response response = dl.get(baseUrl
|
||||
|
|
@ -95,8 +111,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
+ getId() + "/description");
|
||||
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
|
||||
text = JsonUtils.getString(jsonObject, "description");
|
||||
} catch (ReCaptchaException | IOException | JsonParserException e) {
|
||||
e.printStackTrace();
|
||||
} catch (final IOException | ReCaptchaException | JsonParserException ignored) {
|
||||
// Something went wrong when getting the full description, use the shortened one
|
||||
}
|
||||
}
|
||||
return new Description(text, Description.MARKDOWN);
|
||||
|
|
@ -119,8 +135,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public long getTimeStamp() throws ParsingException {
|
||||
final long timestamp =
|
||||
getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
||||
final long timestamp = getTimestampSeconds(
|
||||
"((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
||||
|
||||
if (timestamp == -2) {
|
||||
// regex for timestamp was not found
|
||||
|
|
@ -148,10 +164,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
final String name = JsonUtils.getString(json, "account.name");
|
||||
final String host = JsonUtils.getString(json, "account.host");
|
||||
return getService().getChannelLHFactory()
|
||||
.fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
|
||||
final String name = JsonUtils.getString(json, ACCOUNT_NAME);
|
||||
final String host = JsonUtils.getString(json, ACCOUNT_HOST);
|
||||
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl)
|
||||
.getUrl();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -199,77 +215,51 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() {
|
||||
return json.getArray("streamingPlaylists").getObject(0).getString("playlistUrl");
|
||||
assertPageFetched();
|
||||
|
||||
if (getStreamType() == StreamType.VIDEO_STREAM
|
||||
&& !isNullOrEmpty(json.getObject(FILES))) {
|
||||
return json.getObject(FILES).getString(PLAYLIST_URL, EMPTY_STRING);
|
||||
} else {
|
||||
return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL,
|
||||
EMPTY_STRING);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
return Collections.emptyList();
|
||||
public List<AudioStream> getAudioStreams() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
||||
/*
|
||||
Some videos have audio streams, some videos don't have audio streams.
|
||||
So an audio stream may be available if a video stream is available.
|
||||
Audio streams are also not returned as separated streams for livestreams.
|
||||
That's why the extraction of audio streams is only run when there are video streams
|
||||
extracted and when the content is not a livestream.
|
||||
*/
|
||||
if (audioStreams.isEmpty() && videoStreams.isEmpty()
|
||||
&& getStreamType() == StreamType.VIDEO_STREAM) {
|
||||
getStreams();
|
||||
}
|
||||
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
// mp4
|
||||
try {
|
||||
videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files")));
|
||||
} catch (final Exception ignored) { }
|
||||
|
||||
// HLS
|
||||
try {
|
||||
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
|
||||
for (final Object p : streamingPlaylists) {
|
||||
if (!(p instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject playlist = (JsonObject) p;
|
||||
videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files")));
|
||||
if (videoStreams.isEmpty()) {
|
||||
if (getStreamType() == StreamType.VIDEO_STREAM) {
|
||||
getStreams();
|
||||
} else {
|
||||
extractLiveVideoStreams();
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams", e);
|
||||
}
|
||||
|
||||
if (getStreamType() == StreamType.LIVE_STREAM) {
|
||||
videoStreams.add(new VideoStream(getHlsUrl(), MediaFormat.MPEG_4, "720p"));
|
||||
}
|
||||
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams)
|
||||
throws ParsingException {
|
||||
try {
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
for (final Object s : streams) {
|
||||
if (!(s instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject stream = (JsonObject) s;
|
||||
final String url;
|
||||
if (stream.has("fileDownloadUrl")) {
|
||||
url = JsonUtils.getString(stream, "fileDownloadUrl");
|
||||
} else {
|
||||
url = JsonUtils.getString(stream, "fileUrl");
|
||||
}
|
||||
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
|
||||
final String resolution = JsonUtils.getString(stream, "resolution.label");
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final VideoStream videoStream
|
||||
= new VideoStream(url, torrentUrl, format, resolution);
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
return videoStreams;
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams from array");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
return Collections.emptyList();
|
||||
|
|
@ -284,13 +274,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
|
||||
final List<SubtitlesStream> filteredSubs = new ArrayList<>();
|
||||
for (final SubtitlesStream sub : subtitles) {
|
||||
if (sub.getFormat() == format) {
|
||||
filteredSubs.add(sub);
|
||||
}
|
||||
}
|
||||
return filteredSubs;
|
||||
return subtitles.stream()
|
||||
.filter(sub -> sub.getFormat() == format)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -304,8 +290,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
final List<String> tags = getTags();
|
||||
final String apiUrl;
|
||||
if (tags.isEmpty()) {
|
||||
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name")
|
||||
+ "@" + JsonUtils.getString(json, "account.host")
|
||||
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, ACCOUNT_NAME)
|
||||
+ "@" + JsonUtils.getString(json, ACCOUNT_HOST)
|
||||
+ "/videos?start=0&count=8";
|
||||
} else {
|
||||
apiUrl = getRelatedItemsUrl(tags);
|
||||
|
|
@ -314,7 +300,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
if (Utils.isBlank(apiUrl)) {
|
||||
return null;
|
||||
} else {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
|
||||
getServiceId());
|
||||
getStreamsFromApi(collector, apiUrl);
|
||||
return collector;
|
||||
}
|
||||
|
|
@ -332,11 +319,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
try {
|
||||
return JsonUtils.getString(json, "support");
|
||||
} catch (final ParsingException e) {
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
}
|
||||
|
||||
private String getRelatedItemsUrl(final List<String> tags) throws UnsupportedEncodingException {
|
||||
@Nonnull
|
||||
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
|
||||
throws UnsupportedEncodingException {
|
||||
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
|
||||
final StringBuilder params = new StringBuilder();
|
||||
params.append("start=0&count=8&sort=-createdAt");
|
||||
|
|
@ -348,7 +337,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl)
|
||||
throws ReCaptchaException, IOException, ParsingException {
|
||||
throws IOException, ReCaptchaException, ParsingException {
|
||||
final Response response = getDownloader().get(apiUrl);
|
||||
JsonObject relatedVideosJson = null;
|
||||
if (response != null && !Utils.isBlank(response.responseBody())) {
|
||||
|
|
@ -365,21 +354,20 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
|
||||
final JsonObject jsonObject)
|
||||
throws ParsingException {
|
||||
final JsonObject jsonObject) throws ParsingException {
|
||||
final JsonArray contents;
|
||||
try {
|
||||
contents = (JsonArray) JsonUtils.getValue(jsonObject, "data");
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("unable to extract related videos", e);
|
||||
throw new ParsingException("Could not extract related videos", e);
|
||||
}
|
||||
|
||||
for (final Object c : contents) {
|
||||
if (c instanceof JsonObject) {
|
||||
final JsonObject item = (JsonObject) c;
|
||||
final PeertubeStreamInfoItemExtractor extractor
|
||||
= new PeertubeStreamInfoItemExtractor(item, baseUrl);
|
||||
//do not add the same stream in related streams
|
||||
final PeertubeStreamInfoItemExtractor extractor =
|
||||
new PeertubeStreamInfoItemExtractor(item, baseUrl);
|
||||
// Do not add the same stream in related streams
|
||||
if (!extractor.getUrl().equals(getUrl())) {
|
||||
collector.commit(extractor);
|
||||
}
|
||||
|
|
@ -395,7 +383,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
if (response != null) {
|
||||
setInitialData(response.responseBody());
|
||||
} else {
|
||||
throw new ExtractionException("Unable to extract PeerTube channel data");
|
||||
throw new ExtractionException("Could not extract PeerTube channel data");
|
||||
}
|
||||
|
||||
loadSubtitles();
|
||||
|
|
@ -405,10 +393,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
try {
|
||||
json = JsonParser.object().from(responseBody);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ExtractionException("Unable to extract PeerTube stream data", e);
|
||||
throw new ExtractionException("Could not extract PeerTube stream data", e);
|
||||
}
|
||||
if (json == null) {
|
||||
throw new ExtractionException("Unable to extract PeerTube stream data");
|
||||
throw new ExtractionException("Could not extract PeerTube stream data");
|
||||
}
|
||||
PeertubeParsingHelper.validate(json);
|
||||
}
|
||||
|
|
@ -429,16 +417,253 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
final String ext = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat fmt = MediaFormat.getFromSuffix(ext);
|
||||
if (fmt != null && !isNullOrEmpty(languageCode)) {
|
||||
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
|
||||
subtitles.add(new SubtitlesStream.Builder()
|
||||
.setContent(url, true)
|
||||
.setMediaFormat(fmt)
|
||||
.setLanguageCode(languageCode)
|
||||
.setAutoGenerated(false)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
// ignore all exceptions
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore all exceptions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractLiveVideoStreams() throws ParsingException {
|
||||
try {
|
||||
final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
|
||||
for (final Object s : streamingPlaylists) {
|
||||
if (!(s instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject stream = (JsonObject) s;
|
||||
// Don't use the containsSimilarStream method because it will always return false
|
||||
// so if there are multiples HLS URLs returned, only the first will be extracted in
|
||||
// this case.
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(String.valueOf(stream.getInt("id", -1)))
|
||||
.setContent(stream.getString(PLAYLIST_URL, EMPTY_STRING), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(EMPTY_STRING)
|
||||
.setMediaFormat(MediaFormat.MPEG_4)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.build());
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void getStreams() throws ParsingException {
|
||||
// Progressive streams
|
||||
getStreamsFromArray(json.getArray(FILES), EMPTY_STRING);
|
||||
|
||||
// HLS streams
|
||||
try {
|
||||
final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
|
||||
for (final Object p : streamingPlaylists) {
|
||||
if (!(p instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject playlist = (JsonObject) p;
|
||||
final String playlistUrl = playlist.getString(PLAYLIST_URL);
|
||||
getStreamsFromArray(playlist.getArray(FILES), playlistUrl);
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get streams", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void getStreamsFromArray(@Nonnull final JsonArray streams,
|
||||
final String playlistUrl) throws ParsingException {
|
||||
try {
|
||||
/*
|
||||
Starting with version 3.4.0 of PeerTube, HLS playlist of stream resolutions contain the
|
||||
UUID of the stream, so we can't use the same system to get HLS playlist URL of streams
|
||||
without fetching the master playlist.
|
||||
These UUIDs are the same that the ones returned into the fileUrl and fileDownloadUrl
|
||||
strings.
|
||||
*/
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl)
|
||||
&& playlistUrl.endsWith("-master.m3u8");
|
||||
|
||||
for (final Object s : streams) {
|
||||
if (!(s instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final JsonObject stream = (JsonObject) s;
|
||||
final String resolution = JsonUtils.getString(stream, "resolution.label");
|
||||
final String url;
|
||||
final String idSuffix;
|
||||
|
||||
// Extract stream version of streams first
|
||||
if (stream.has(FILE_URL)) {
|
||||
url = JsonUtils.getString(stream, FILE_URL);
|
||||
idSuffix = FILE_URL;
|
||||
} else {
|
||||
url = JsonUtils.getString(stream, FILE_DOWNLOAD_URL);
|
||||
idSuffix = FILE_DOWNLOAD_URL;
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(url)) {
|
||||
// Not a valid stream URL
|
||||
return;
|
||||
}
|
||||
|
||||
if (resolution.toLowerCase().contains("audio")) {
|
||||
// An audio stream
|
||||
addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
|
||||
idSuffix, url, playlistUrl);
|
||||
} else {
|
||||
// A video stream
|
||||
addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
|
||||
idSuffix, url, playlistUrl);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get streams from array", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getHlsPlaylistUrlFromFragmentedFileUrl(
|
||||
@Nonnull final JsonObject streamJsonObject,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String format,
|
||||
@Nonnull final String url) throws ParsingException {
|
||||
final String streamUrl;
|
||||
if (FILE_DOWNLOAD_URL.equals(idSuffix)) {
|
||||
streamUrl = JsonUtils.getString(streamJsonObject, FILE_URL);
|
||||
} else {
|
||||
streamUrl = url;
|
||||
}
|
||||
return streamUrl.replace("-fragmented." + format, ".m3u8");
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject,
|
||||
@Nonnull final String playlistUrl)
|
||||
throws ParsingException {
|
||||
return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject,
|
||||
RESOLUTION_ID).toString());
|
||||
}
|
||||
|
||||
private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject,
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams,
|
||||
@Nonnull final String resolution,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String url,
|
||||
@Nullable final String playlistUrl) throws ParsingException {
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final String id = resolution + "-" + extension;
|
||||
|
||||
// Add progressive HTTP streams first
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
|
||||
.setContent(url, true)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
|
||||
// Then add HLS streams
|
||||
if (!isNullOrEmpty(playlistUrl)) {
|
||||
final String hlsStreamUrl;
|
||||
if (isInstanceUsingRandomUuidsForHlsStreams) {
|
||||
hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
|
||||
extension, url);
|
||||
|
||||
} else {
|
||||
hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
|
||||
}
|
||||
final AudioStream audioStream = new AudioStream.Builder()
|
||||
.setId(id + "-" + DeliveryMethod.HLS)
|
||||
.setContent(hlsStreamUrl, true)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.setBaseUrl(playlistUrl)
|
||||
.build();
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
}
|
||||
|
||||
// Add finally torrent URLs
|
||||
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
|
||||
if (!isNullOrEmpty(torrentUrl)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
|
||||
.setContent(torrentUrl, true)
|
||||
.setDeliveryMethod(DeliveryMethod.TORRENT)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject,
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams,
|
||||
@Nonnull final String resolution,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String url,
|
||||
@Nullable final String playlistUrl) throws ParsingException {
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final String id = resolution + "-" + extension;
|
||||
|
||||
// Add progressive HTTP streams first
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
|
||||
.setContent(url, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.build());
|
||||
|
||||
// Then add HLS streams
|
||||
if (!isNullOrEmpty(playlistUrl)) {
|
||||
final String hlsStreamUrl;
|
||||
if (isInstanceUsingRandomUuidsForHlsStreams) {
|
||||
hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
|
||||
extension, url);
|
||||
} else {
|
||||
hlsStreamUrl = playlistUrl.replace("master", JsonUtils.getNumber(
|
||||
streamJsonObject, RESOLUTION_ID).toString());
|
||||
}
|
||||
|
||||
final VideoStream videoStream = new VideoStream.Builder()
|
||||
.setId(id + "-" + DeliveryMethod.HLS)
|
||||
.setContent(hlsStreamUrl, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.setBaseUrl(playlistUrl)
|
||||
.build();
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
|
||||
// Add finally torrent URLs
|
||||
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
|
||||
if (!isNullOrEmpty(torrentUrl)) {
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
|
||||
.setContent(torrentUrl, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setDeliveryMethod(DeliveryMethod.TORRENT)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
|
|
@ -448,7 +673,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return JsonUtils.getString(json, "account.host");
|
||||
return JsonUtils.getString(json, ACCOUNT_HOST);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
|
||||
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId;
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
|
@ -26,6 +29,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
|
|||
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
|
@ -58,13 +62,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
final String policy = track.getString("policy", EMPTY_STRING);
|
||||
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
|
||||
isAvailable = false;
|
||||
|
||||
if (policy.equals("SNIP")) {
|
||||
throw new SoundCloudGoPlusContentException();
|
||||
}
|
||||
|
||||
if (policy.equals("BLOCK")) {
|
||||
throw new GeographicRestrictionException(
|
||||
"This track is not available in user's country");
|
||||
}
|
||||
|
||||
throw new ContentNotAvailableException("Content not available: policy " + policy);
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +79,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getId() {
|
||||
return track.getInt("id") + EMPTY_STRING;
|
||||
return String.valueOf(track.getInt("id"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -162,17 +169,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
// Streams can be streamable and downloadable - or explicitly not.
|
||||
// For playing the track, it is only necessary to have a streamable track.
|
||||
// If this is not the case, this track might not be published yet.
|
||||
// If audio streams were calculated, return the calculated result
|
||||
if (!track.getBoolean("streamable") || !isAvailable) {
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
try {
|
||||
final JsonArray transcodings = track.getObject("media").getArray("transcodings");
|
||||
if (transcodings != null) {
|
||||
if (!isNullOrEmpty(transcodings)) {
|
||||
// Get information about what stream formats are available
|
||||
extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
|
||||
audioStreams);
|
||||
}
|
||||
extractDownloadableFileIfAvailable(audioStreams);
|
||||
} catch (final NullPointerException e) {
|
||||
throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
|
||||
}
|
||||
|
|
@ -180,7 +189,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
return audioStreams;
|
||||
}
|
||||
|
||||
private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) {
|
||||
private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
|
||||
boolean presence = false;
|
||||
for (final Object transcoding : transcodings) {
|
||||
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
|
||||
|
|
@ -195,48 +204,68 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Nonnull
|
||||
private static String getTranscodingUrl(final String endpointUrl,
|
||||
final String protocol)
|
||||
private String getTranscodingUrl(final String endpointUrl,
|
||||
final String protocol)
|
||||
throws IOException, ExtractionException {
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
final String apiStreamUrl = endpointUrl + "?client_id="
|
||||
+ SoundcloudParsingHelper.clientId();
|
||||
+ clientId();
|
||||
final String response = downloader.get(apiStreamUrl).responseBody();
|
||||
final JsonObject urlObject;
|
||||
try {
|
||||
urlObject = JsonParser.object().from(response);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ParsingException("Could not parse streamable url", e);
|
||||
throw new ParsingException("Could not parse streamable URL", e);
|
||||
}
|
||||
|
||||
final String urlString = urlObject.getString("url");
|
||||
|
||||
if (protocol.equals("progressive")) {
|
||||
return urlString;
|
||||
} else if (protocol.equals("hls")) {
|
||||
try {
|
||||
return getSingleUrlFromHlsManifest(urlString);
|
||||
} catch (final ParsingException ignored) {
|
||||
}
|
||||
return getSingleUrlFromHlsManifest(urlString);
|
||||
}
|
||||
|
||||
// else, unknown protocol
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
private static void extractAudioStreams(final JsonArray transcodings,
|
||||
final boolean mp3ProgressiveInStreams,
|
||||
final List<AudioStream> audioStreams) {
|
||||
@Nullable
|
||||
private String getDownloadUrl(@Nonnull final String trackId)
|
||||
throws IOException, ExtractionException {
|
||||
final Downloader dl = NewPipe.getDownloader();
|
||||
final JsonObject downloadJsonObject;
|
||||
|
||||
final String response = dl.get(SOUNDCLOUD_API_V2_URL + "tracks/" + trackId
|
||||
+ "/download" + "?client_id=" + clientId()).responseBody();
|
||||
try {
|
||||
downloadJsonObject = JsonParser.object().from(response);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ParsingException("Could not parse download URL", e);
|
||||
}
|
||||
final String redirectUri = downloadJsonObject.getString("redirectUri");
|
||||
if (!isNullOrEmpty(redirectUri)) {
|
||||
return redirectUri;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
|
||||
final boolean mp3ProgressiveInStreams,
|
||||
final List<AudioStream> audioStreams) {
|
||||
for (final Object transcoding : transcodings) {
|
||||
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
|
||||
final String url = transcodingJsonObject.getString("url");
|
||||
if (isNullOrEmpty(url)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final String mediaUrl;
|
||||
final String preset = transcodingJsonObject.getString("preset");
|
||||
final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN);
|
||||
final String protocol = transcodingJsonObject.getObject("format")
|
||||
.getString("protocol");
|
||||
MediaFormat mediaFormat = null;
|
||||
int bitrate = 0;
|
||||
int averageBitrate = UNKNOWN_BITRATE;
|
||||
if (preset.contains("mp3")) {
|
||||
// Don't add the MP3 HLS stream if there is a progressive stream present
|
||||
// because the two have the same bitrate
|
||||
|
|
@ -244,36 +273,75 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
continue;
|
||||
}
|
||||
mediaFormat = MediaFormat.MP3;
|
||||
bitrate = 128;
|
||||
averageBitrate = 128;
|
||||
} else if (preset.contains("opus")) {
|
||||
mediaFormat = MediaFormat.OPUS;
|
||||
bitrate = 64;
|
||||
averageBitrate = 64;
|
||||
}
|
||||
|
||||
if (mediaFormat != null) {
|
||||
try {
|
||||
mediaUrl = getTranscodingUrl(url, protocol);
|
||||
if (!mediaUrl.isEmpty()) {
|
||||
audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate));
|
||||
try {
|
||||
mediaUrl = getTranscodingUrl(url, protocol);
|
||||
if (!mediaUrl.isEmpty()) {
|
||||
final AudioStream audioStream = new AudioStream.Builder()
|
||||
.setId(preset)
|
||||
.setContent(mediaUrl, true)
|
||||
.setMediaFormat(mediaFormat)
|
||||
.setAverageBitrate(averageBitrate)
|
||||
.build();
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
// something went wrong when parsing this transcoding, don't add it to
|
||||
// audioStreams
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
// Something went wrong when parsing this transcoding, don't add it to the
|
||||
// audioStreams
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the downloadable format if it is available.
|
||||
*
|
||||
* <p>
|
||||
* A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
|
||||
* we can download it: if the value of the {@code has_download_left} boolean is true, the track
|
||||
* can be downloaded; otherwise not.
|
||||
* </p>
|
||||
*
|
||||
* @param audioStreams the audio streams to which add the downloadable file
|
||||
*/
|
||||
public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
|
||||
if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
|
||||
try {
|
||||
final String downloadUrl = getDownloadUrl(getId());
|
||||
if (!isNullOrEmpty(downloadUrl)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId("original-format")
|
||||
.setContent(downloadUrl, true)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
// If something went wrong when trying to get the download URL, ignore the
|
||||
// exception throw because this "stream" is not necessary to play the track
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
|
||||
*
|
||||
* <p>
|
||||
* This method downloads the provided manifest URL, find all web occurrences in the manifest,
|
||||
* get the last segment URL, changes its segment range to {@code 0/track-length} and return
|
||||
* this string.
|
||||
* </p>
|
||||
*
|
||||
* @param hlsManifestUrl the URL of the manifest to be parsed
|
||||
* @return a single URL that contains a range equal to the length of the track
|
||||
*/
|
||||
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
|
||||
@Nonnull
|
||||
private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
|
||||
throws ParsingException {
|
||||
final Downloader dl = NewPipe.getDownloader();
|
||||
final String hlsManifestResponse;
|
||||
|
|
@ -326,7 +394,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
|
||||
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
|
||||
+ "/related?client_id=" + urlEncode(clientId());
|
||||
|
||||
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
|
||||
return collector;
|
||||
|
|
@ -355,19 +423,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
// Tags are separated by spaces, but they can be multiple words escaped by quotes "
|
||||
final String[] tagList = track.getString("tag_list").split(" ");
|
||||
final List<String> tags = new ArrayList<>();
|
||||
String escapedTag = "";
|
||||
final StringBuilder escapedTag = new StringBuilder();
|
||||
boolean isEscaped = false;
|
||||
for (final String tag : tagList) {
|
||||
if (tag.startsWith("\"")) {
|
||||
escapedTag += tag.replace("\"", "");
|
||||
escapedTag.append(tag.replace("\"", ""));
|
||||
isEscaped = true;
|
||||
} else if (isEscaped) {
|
||||
if (tag.endsWith("\"")) {
|
||||
escapedTag += " " + tag.replace("\"", "");
|
||||
escapedTag.append(" ").append(tag.replace("\"", ""));
|
||||
isEscaped = false;
|
||||
tags.add(escapedTag);
|
||||
tags.add(escapedTag.toString());
|
||||
} else {
|
||||
escapedTag += " " + tag;
|
||||
escapedTag.append(" ").append(tag);
|
||||
}
|
||||
} else if (!tag.isEmpty()) {
|
||||
tags.add(tag);
|
||||
|
|
|
|||
Loading…
Reference in New Issue