be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #859 from AudricV/delivery-methods-fixes-and-improvements 

Fix extraction of some properties in ItagItems for YouTube livestreams and post-live streams and remove completely SoundCloud HLS workaround
This commit is contained in:
Stypox 2022-06-17 15:46:13 +02:00 committed by GitHub
parent c8a77da2ab 301a795ed3
commit 1b51eab664
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -22,7 +21,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -245,61 +243,30 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return;
}
final String preset = transcoding.getString("preset", ID_UNKNOWN);
final String protocol = transcoding.getObject("format").getString("protocol");
final AudioStream.Builder builder = new AudioStream.Builder()
.setId(preset);
try {
// streamUrl can be either the MP3 progressive stream URL or the
// manifest URL of the HLS MP3 stream (if there is no MP3 progressive
// stream, see above)
final String streamUrl = getTranscodingUrl(url);
final String preset = transcoding.getString("preset", ID_UNKNOWN);
final String protocol = transcoding.getObject("format")
.getString("protocol");
final AudioStream.Builder builder = new AudioStream.Builder()
.setId(preset);
final boolean isHls = protocol.equals("hls");
if (isHls) {
builder.setDeliveryMethod(DeliveryMethod.HLS);
}
builder.setContent(getTranscodingUrl(url), true);
if (preset.contains("mp3")) {
// Don't add the MP3 HLS stream if there is a progressive stream
// present because the two have the same bitrate
final boolean isHls = protocol.equals("hls");
// present because both have the same bitrate
if (mp3ProgressiveInStreams && isHls) {
return;
}
builder.setMediaFormat(MediaFormat.MP3);
builder.setAverageBitrate(128);
if (isHls) {
builder.setDeliveryMethod(DeliveryMethod.HLS);
builder.setContent(streamUrl, true);
final AudioStream hlsStream = builder.build();
if (!Stream.containSimilarStream(hlsStream, audioStreams)) {
audioStreams.add(hlsStream);
}
final String progressiveHlsUrl =
getSingleUrlFromHlsManifest(streamUrl);
builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP);
builder.setContent(progressiveHlsUrl, true);
final AudioStream progressiveHlsStream = builder.build();
if (!Stream.containSimilarStream(
progressiveHlsStream, audioStreams)) {
audioStreams.add(progressiveHlsStream);
}
// The MP3 HLS stream has been added in both versions (HLS and
// progressive with the manifest parsing trick), so we need to
// continue (otherwise the code would try to add again the stream,
// which would be not added because the containsSimilarStream
// method would return false and an audio stream object would be
// created for nothing)
return;
} else {
builder.setContent(streamUrl, true);
}
} else if (preset.contains("opus")) {
// The HLS manifest trick doesn't work for opus streams
builder.setContent(streamUrl, true);
builder.setMediaFormat(MediaFormat.OPUS);
builder.setAverageBitrate(64);
builder.setDeliveryMethod(DeliveryMethod.HLS);
@ -352,47 +319,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
}
/**
* Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams.
*
* <p>
* This method downloads the provided manifest URL, finds all web occurrences in the manifest,
* gets the last segment URL, changes its segment range to {@code 0/track-length}, and return
* this as a string.
* </p>
*
* <p>
* This was working before for Opus streams, but has been broken by SoundCloud.
* </p>
*
* @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track
*/
@Nonnull
private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
throws ParsingException {
final String hlsManifestResponse;
try {
hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody();
} catch (final IOException | ReCaptchaException e) {
throw new ParsingException("Could not get SoundCloud HLS manifest");
}
final String[] lines = hlsManifestResponse.split("\\r?\\n");
for (int l = lines.length - 1; l >= 0; l--) {
final String line = lines[l];
// Get the last URL from manifest, because it contains the range of the stream
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) {
final String[] hlsLastRangeUrlArray = line.split("/");
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
+ "/" + hlsLastRangeUrlArray[6];
}
}
throw new ParsingException("Could not get any URL from HLS manifest");
}
private static String urlEncode(final String value) {
try {
return URLEncoder.encode(value, UTF_8);

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1358,13 +1358,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) {
itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec"));
} else if (itagType == ItagItem.ItagType.VIDEO
|| itagType == ItagItem.ItagType.VIDEO_ONLY) {
}
if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
itagItem.setFps(formatData.getInt("fps"));
} else if (itagType == ItagItem.ItagType.AUDIO) {
// YouTube return the audio sample rate as a string
itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
itagItem.setAudioChannels(formatData.getInt("audioChannels"));
itagItem.setAudioChannels(formatData.getInt("audioChannels",
// Most audio streams have two audio channels, so use this value if the real
// count cannot be extracted
// Doing this prevents an exception when generating the
// AudioChannelConfiguration element of DASH manifests of audio streams in
// YoutubeDashManifestCreatorUtils
2));
}
// YouTube return the content length and the approximate duration as strings