Merge pull request #526 from TiA4f8R/snd-hls-workaround

Support SoundCloud HLS-only tracks by using a workaround

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -10,10 +10,10 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -34,6 +34,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.*;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
+    private boolean isAvailable = true;
 
     public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
         super(service, linkHandler);
@@ -43,8 +44,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
 
-        String policy = track.getString("policy", EMPTY_STRING);
+        final String policy = track.getString("policy", EMPTY_STRING);
         if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
+            isAvailable = false;
             if (policy.equals("SNIP")) {
                 throw new SoundCloudGoPlusContentException();
             }
@@ -181,62 +183,143 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+    public List<AudioStream> getAudioStreams() throws ExtractionException {
-        List<AudioStream> audioStreams = new ArrayList<>();
+        final List<AudioStream> audioStreams = new ArrayList<>();
-        final Downloader dl = NewPipe.getDownloader();
 
         // Streams can be streamable and downloadable - or explicitly not.
         // For playing the track, it is only necessary to have a streamable track.
         // If this is not the case, this track might not be published yet.
-        if (!track.getBoolean("streamable")) return audioStreams;
+        if (!track.getBoolean("streamable") || !isAvailable) return audioStreams;
 
         try {
             final JsonArray transcodings = track.getObject("media").getArray("transcodings");
-
+            if (transcodings != null) {
-            // get information about what stream formats are available
+                // Get information about what stream formats are available
-            for (Object transcoding : transcodings) {
+                extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
-
+                        audioStreams);
-                final JsonObject t = (JsonObject) transcoding;
-                String url = t.getString("url");
-
-                if (!isNullOrEmpty(url)) {
-
-                    // We can only play the mp3 format, but not handle m3u playlists / streams.
-                    // what about Opus?
-                    if (t.getString("preset").contains("mp3")
-                            && t.getObject("format").getString("protocol").equals("progressive")) {
-                        // This url points to the endpoint which generates a unique and short living url to the stream.
-                        // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream)
-                        url += "?client_id=" + SoundcloudParsingHelper.clientId();
-                        final String res = dl.get(url).responseBody();
-
-                        try {
-                            JsonObject mp3UrlObject = JsonParser.object().from(res);
-                            // Links in this file are also only valid for a short period.
-                            audioStreams.add(new AudioStream(mp3UrlObject.getString("url"),
-                                    MediaFormat.MP3, 128));
-                        } catch (JsonParserException e) {
-                            throw new ParsingException("Could not parse streamable url", e);
-                        }
-                    }
-                }
             }
-
+        } catch (final NullPointerException e) {
-        } catch (NullPointerException e) {
+            throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
-            throw new ExtractionException("Could not get SoundCloud's track audio url", e);
-        }
-
-        if (audioStreams.isEmpty()) {
-            throw new ContentNotSupportedException("HLS audio streams are not yet supported");
         }
 
         return audioStreams;
     }
 
-    private static String urlEncode(String value) {
+    private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) {
+        boolean presence = false;
+        for (final Object transcoding : transcodings) {
+            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
+            if (transcodingJsonObject.getString("preset").contains("mp3") &&
+                    transcodingJsonObject.getObject("format").getString("protocol")
+                            .equals("progressive")) {
+                presence = true;
+                break;
+            }
+        }
+        return presence;
+    }
+
+    @Nonnull
+    private static String getTranscodingUrl(final String endpointUrl, final String protocol) throws IOException, ExtractionException {
+        final Downloader downloader = NewPipe.getDownloader();
+        final String apiStreamUrl = endpointUrl + "?client_id=" + SoundcloudParsingHelper.clientId();
+        final String response = downloader.get(apiStreamUrl).responseBody();
+        final JsonObject urlObject;
+        try {
+            urlObject = JsonParser.object().from(response);
+        } catch (final JsonParserException e) {
+            throw new ParsingException("Could not parse streamable url", e);
+        }
+        final String urlString = urlObject.getString("url");
+
+        if (protocol.equals("progressive")) {
+            return urlString;
+        } else if (protocol.equals("hls")) {
+            try {
+                return getSingleUrlFromHlsManifest(urlString);
+            } catch (final ParsingException ignored) {
+            }
+        }
+        // else, unknown protocol
+        return "";
+    }
+
+    private static void extractAudioStreams(final JsonArray transcodings,
+                                            final boolean mp3ProgressiveInStreams,
+                                            final List<AudioStream> audioStreams) {
+        for (final Object transcoding : transcodings) {
+            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
+            final String url = transcodingJsonObject.getString("url");
+            if (isNullOrEmpty(url)) {
+                continue;
+            }
+            final String mediaUrl;
+            final String preset = transcodingJsonObject.getString("preset");
+            final String protocol = transcodingJsonObject.getObject("format").getString("protocol");
+            MediaFormat mediaFormat = null;
+            int bitrate = 0;
+            if (preset.contains("mp3")) {
+                // Don't add the MP3 HLS stream if there is a progressive stream present
+                // because the two have the same bitrate
+                if (mp3ProgressiveInStreams && protocol.equals("hls")) {
+                    continue;
+                }
+                mediaFormat = MediaFormat.MP3;
+                bitrate = 128;
+            } else if (preset.contains("opus")) {
+                mediaFormat = MediaFormat.OPUS;
+                bitrate = 64;
+            }
+
+            if (mediaFormat != null) {
+                try {
+                    mediaUrl = getTranscodingUrl(url, protocol);
+                    if (!mediaUrl.isEmpty()) {
+                        audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate));
+                    }
+                } catch (final Exception ignored) {
+                    // something went wrong when parsing this transcoding, don't add it to
+                    // audioStreams
+                }
+            }
+        }
+    }
+
+    /** Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
+     * <p>
+     * This method downloads the provided manifest URL, find all web occurrences in the manifest,
+     * get the last segment URL, changes its segment range to {@code 0/track-length} and return
+     * this string.
+     * @param  hlsManifestUrl the URL of the manifest to be parsed
+     * @return a single URL that contains a range equal to the length of the track
+     */
+    private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException {
+        final Downloader dl = NewPipe.getDownloader();
+        final String hlsManifestResponse;
+
+        try {
+            hlsManifestResponse = dl.get(hlsManifestUrl).responseBody();
+        } catch (final IOException | ReCaptchaException e) {
+            throw new ParsingException("Could not get SoundCloud HLS manifest");
+        }
+
+        final String[] lines = hlsManifestResponse.split("\\r?\\n");
+        for (int l = lines.length - 1; l >= 0; l--) {
+            final String line = lines[l];
+            // get the last URL from manifest, because it contains the range of the stream
+            if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
+                final String[] hlsLastRangeUrlArray = line.split("/");
+                return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/"
+                        + hlsLastRangeUrlArray[6];
+            }
+        }
+        throw new ParsingException("Could not get any URL from HLS manifest");
+    }
+
+    private static String urlEncode(final String value) {
         try {
             return URLEncoder.encode(value, UTF_8);
-        } catch (UnsupportedEncodingException e) {
+        } catch (final UnsupportedEncodingException e) {
             throw new IllegalStateException(e);
         }
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -1,15 +1,15 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
 import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -19,12 +19,14 @@ import java.util.List;
 
 import javax.annotation.Nullable;
 
+import static junit.framework.TestCase.assertEquals;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 
 public class SoundcloudStreamExtractorTest {
     private static final String SOUNDCLOUD = "https://soundcloud.com/";
 
-    @Ignore("Ignore until #526 is merged. Throwing the ContentNotSupportedException is wrong and going to be fixed by that PR.")
     public static class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest {
         private static final String ID = "one-touch";
         private static final String UPLOADER = SOUNDCLOUD + "jessglynne";
@@ -59,6 +61,7 @@ public class SoundcloudStreamExtractorTest {
         @Nullable @Override public String expectedTextualUploadDate() { return "2019-05-16 16:28:45"; }
         @Override public long expectedLikeCountAtLeast() { return -1; }
         @Override public long expectedDislikeCountAtLeast() { return -1; }
+        @Override public boolean expectedHasAudioStreams() { return false; }
         @Override public boolean expectedHasVideoStreams() { return false; }
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
@@ -100,7 +103,9 @@ public class SoundcloudStreamExtractorTest {
         @Nullable @Override public String expectedTextualUploadDate() { return "2016-11-11 01:16:37"; }
         @Override public long expectedLikeCountAtLeast() { return -1; }
         @Override public long expectedDislikeCountAtLeast() { return -1; }
+        @Override public boolean expectedHasAudioStreams() { return false; }
         @Override public boolean expectedHasVideoStreams() { return false; }
+        @Override public boolean expectedHasRelatedStreams() { return false; }
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
@@ -143,6 +148,25 @@ public class SoundcloudStreamExtractorTest {
         @Override public boolean expectedHasSubtitles() { return false; }
         @Override public boolean expectedHasFrames() { return false; }
         @Override public int expectedStreamSegmentsCount() { return 0; }
-    }
 
+        @Override
+        @Test
+        public void testAudioStreams() throws Exception {
+            super.testAudioStreams();
+            final List<AudioStream> audioStreams = extractor.getAudioStreams();
+            assertEquals(2, audioStreams.size());
+            for (final AudioStream audioStream : audioStreams) {
+                final String mediaUrl = audioStream.getUrl();
+                if (audioStream.getFormat() == MediaFormat.OPUS) {
+                    // assert that it's an OPUS 64 kbps media URL with a single range which comes from an HLS SoundCloud CDN
+                    assertThat(mediaUrl, containsString("-hls-opus-media.sndcdn.com"));
+                    assertThat(mediaUrl, containsString(".64.opus"));
+                }
+                if (audioStream.getFormat() == MediaFormat.MP3) {
+                    // assert that it's a MP3 128 kbps media URL which comes from a progressive SoundCloud CDN
+                    assertThat(mediaUrl, containsString("-media.sndcdn.com/bKOA7Pwbut93.128.mp3"));
+                }
+            }
+        }
+    }
 }