be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #293 from Stypox/fix-soundcloud 

[SoundCloud] Fix by migrating to `api-v2` and other improvements
This commit is contained in:
Tobias Groza 2020-03-18 17:49:54 +01:00 committed by GitHub
parent 9eca7df947 222d659d9e
commit 6f03c6e877
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 211 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -44,7 +44,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
}
private void computNextPageAndStreams() throws IOException, ExtractionException {
private void computeNextPageAndStreams() throws IOException, ExtractionException {
collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
@ -69,7 +69,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if (nextPageUrl == null) {
computNextPageAndStreams();
computeNextPageAndStreams();
}
return nextPageUrl;
}
@ -78,7 +78,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if (collector == null) {
computNextPageAndStreams();
computeNextPageAndStreams();
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -108,7 +108,7 @@ public class SoundcloudParsingHelper {
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api.soundcloud.com/resolve"
String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();

99
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -1,8 +1,11 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -12,16 +15,20 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private static final int streamsPerRequestedPage = 15;
private String playlistId;
private JsonObject playlist;
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
private StreamInfoItemsCollector streamInfoItemsCollector;
private String nextPageUrl;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
@ -31,7 +38,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
playlistId = getLinkHandler().getId();
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
@ -55,6 +62,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return playlist.getString("title");
}
@Nullable
@Override
public String getThumbnailUrl() {
String artworkUrl = playlist.getString("artwork_url");
@ -64,21 +72,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
// if it also fails, return null
try {
final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
if (infoItems.getItems().isEmpty()) return null;
for (StreamInfoItem item : infoItems.getItems()) {
final String thumbnailUrl = item.getThumbnailUrl();
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
return thumbnailUrlBetterResolution;
artworkUrl = item.getThumbnailUrl();
if (artworkUrl != null && !artworkUrl.isEmpty()) break;
}
} catch (Exception ignored) {
}
if (artworkUrl == null) {
return null;
}
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
return artworkUrl.replace("large.jpg", "crop.jpg");
}
@Override
@ -110,27 +117,42 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if (streamInfoItemsCollector == null) {
computeStreamsAndNextPageUrl();
computeInitialTracksAndNextPageUrl();
}
return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl);
}
private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id=");
nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId());
nextPageUrlBuilder.append("&ids=");
// Note the "api", NOT "api-v2"
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
JsonArray tracks = playlist.getArray("tracks");
for (Object o : tracks) {
if (o instanceof JsonObject) {
JsonObject track = (JsonObject) o;
if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else {
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure
nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id")));
}
}
}
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing ,
nextPageUrl = nextPageUrlBuilder.toString();
if (nextPageUrl.endsWith("&ids")) {
// there are no other videos
nextPageUrl = "";
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if (nextPageUrl == null) {
computeStreamsAndNextPageUrl();
computeInitialTracksAndNextPageUrl();
}
return nextPageUrl;
}
@ -141,9 +163,36 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
// see computeInitialTracksAndNextPageUrl
final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id="
+ SoundcloudParsingHelper.clientId()
+ "&ids=").length();
final int lengthOfEveryStream = 11;
return new InfoItemsPage<>(collector, nextPageUrl);
String currentPageUrl, nextUrl;
int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage;
if (pageUrl.length() <= lengthMaxStreams) {
currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max
nextUrl = ""; // afterwards the list is complete
} else {
currentPageUrl = pageUrl.substring(0, lengthMaxStreams);
nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams);
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
try {
JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) {
if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
}
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collector, nextUrl);
}
}

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -13,9 +14,15 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -24,6 +31,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -55,14 +64,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getTextualUploadDate() {
return track.getString("created_at");
public String getTextualUploadDate() throws ParsingException {
return track.getString("created_at").replace("T"," ").replace("Z", "");
}
@Nonnull
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate()));
return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at")));
}
@Nonnull
@ -146,24 +155,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
List<AudioStream> audioStreams = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
String response = dl.get(apiUrl, getExtractorLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
// Streams can be streamable and downloadable - or explicitly not.
// For playing the track, it is only necessary to have a streamable track.
// If this is not the case, this track might not be published yet.
if (!responseObject.getBoolean("streamable")) return audioStreams;
if (!track.getBoolean("streamable")) return audioStreams;
try {
JsonArray transcodings = responseObject.getObject("media").getArray("transcodings");
JsonArray transcodings = track.getObject("media").getArray("transcodings");
// get information about what stream formats are available
for (Object transcoding : transcodings) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java
View File

@ -36,7 +36,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
throw new InvalidSourceException(e);
}
String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=200";
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java
View File

@ -24,6 +24,8 @@ public class SoundcloudParsingHelperTest {
public void resolveUrlWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/26057743"));
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/16069159"));
}
@Test

139
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -1,8 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.hamcrest.CoreMatchers;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
@ -11,6 +9,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -70,14 +69,8 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
public void testMoreRelatedItems() {
try {
defaultTestMoreItems(extractor);
} catch (Throwable ignored) {
return;
}
fail("This playlist doesn't have more items, it should throw an error");
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -89,17 +82,17 @@ public class SoundcloudPlaylistExtractorTest {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Ignore
@Test
public void testBannerUrl() {
assertIsSecureUrl(extractor.getBannerUrl());
// SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl());
}
@Test
public void testUploaderUrl() {
final String uploaderUrl = extractor.getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertTrue(uploaderUrl, uploaderUrl.contains("liluzivert"));
assertThat(uploaderUrl, containsString("liluzivert"));
}
@Test
@ -114,7 +107,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testStreamCount() {
assertTrue("Error in the streams count", extractor.getStreamCount() >= 10);
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10);
}
}
@ -181,17 +174,17 @@ public class SoundcloudPlaylistExtractorTest {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Ignore("not implemented")
@Test
public void testBannerUrl() {
assertIsSecureUrl(extractor.getBannerUrl());
// SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl());
}
@Test
public void testUploaderUrl() {
final String uploaderUrl = extractor.getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertThat(uploaderUrl, CoreMatchers.containsString("micky96"));
assertThat(uploaderUrl, containsString("micky96"));
}
@Test
@ -206,7 +199,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testStreamCount() {
assertTrue("Error in the streams count", extractor.getStreamCount() >= 10);
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10);
}
}
@ -227,7 +220,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
@ -264,14 +257,11 @@ public class SoundcloudPlaylistExtractorTest {
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
//TODO: FUCK THIS: This triggers a 500 at sever
@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
@ -286,16 +276,15 @@ public class SoundcloudPlaylistExtractorTest {
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testThumbnailUrl() {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Ignore
@Test
public void testBannerUrl() {
assertIsSecureUrl(extractor.getBannerUrl());
// SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl());
}
@Test
@ -317,7 +306,105 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testStreamCount() {
assertTrue("Error in the streams count", extractor.getStreamCount() >= 3900);
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370);
}
}
public static class SmallPlaylist implements BasePlaylistExtractorTest {
private static SoundcloudPlaylistExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(SoundCloud.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() {
assertEquals("EMPTY PLAYLIST", extractor.getName());
}
@Test
public void testId() {
assertEquals("23483459", extractor.getId());
}
@Test
public void testUrl() throws Exception {
assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws Exception {
assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
try {
defaultTestMoreItems(extractor);
} catch (Throwable ignored) {
return;
}
fail("This playlist doesn't have more items, it should throw an error");
}
/*//////////////////////////////////////////////////////////////////////////
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testThumbnailUrl() {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testBannerUrl() {
// SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl());
}
@Test
public void testUploaderUrl() {
final String uploaderUrl = extractor.getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertThat(uploaderUrl, containsString("breezy-123"));
}
@Test
public void testUploaderName() {
assertEquals("breezy-123", extractor.getUploaderName());
}
@Test
public void testUploaderAvatarUrl() {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testStreamCount() {
assertEquals(2, extractor.getStreamCount());
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -74,7 +74,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2016/07/31 18:18:07 +0000", extractor.getTextualUploadDate());
Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate());
}
@Test