From 9c12dc5609b7d1567e855b90a5a19520a7f6855f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 26 Apr 2021 17:58:30 +0200 Subject: [PATCH] [SoundCloud] Fix SoundCloud ID extraction resolveIdWithEmbedPlayer() does not work anymore because the JSON data has been extracted to an API call. For this reason, replace resolveIdWithEmbedPlayer() with resolveIdWithWidgetApi)( which performs the API call. --- .../soundcloud/SoundcloudParsingHelper.java | 29 +++++++++++++------ .../SoundcloudChannelLinkHandlerFactory.java | 2 +- .../SoundcloudPlaylistLinkHandlerFactory.java | 2 +- .../SoundcloudStreamLinkHandlerFactory.java | 2 +- .../schabi/newpipe/extractor/utils/Utils.java | 9 ++++++ .../SoundcloudParsingHelperTest.java | 6 ++-- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index e92610e42..ecf831d5d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; @@ -140,27 +141,37 @@ public class SoundcloudParsingHelper { } /** - * Fetch the embed player with the url and return the id (like the id from the json api). + * Fetch the widget API with the url and return the id (like the id from the json api). * * @return the resolved id */ - public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException { + public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException { // Remove the tailing slash from URLs due to issues with the SoundCloud API if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1); + // Make URL lower case and remove www. if it exists. + // Without doing this, the widget API does not recognize the URL. + urlString = Utils.removeWWWFromUrl(urlString.toLowerCase()); - URL url; + final URL url; try { url = Utils.stringToURL(urlString); } catch (MalformedURLException e) { throw new IllegalArgumentException("The given URL is not valid"); } - String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody(); - // handle playlists / sets different and get playlist id via uir field in JSON - if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets")) - return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); - return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response); + try { + final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" + + URLEncoder.encode(url.toString(), UTF_8) + + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); + final String response = NewPipe.getDownloader().get(widgetUrl, + SoundCloud.getLocalization()).responseBody(); + final JsonObject o = JsonParser.object().from(response); + return String.valueOf(JsonUtils.getValue(o, "id")); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON response", e); + } catch (ExtractionException e) { + throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e); + } } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java index e49834ebc..c1308a96f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java @@ -23,7 +23,7 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java index 3adade10c..c89e261c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java @@ -22,7 +22,7 @@ public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 48aa836e1..55eafa8a3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -32,7 +32,7 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 82ff58263..baf502ea6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -7,6 +7,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.*; +import java.util.regex.Pattern; public class Utils { @@ -14,6 +15,7 @@ public class Utils { public static final String HTTPS = "https://"; public static final String UTF_8 = "UTF-8"; public static final String EMPTY_STRING = ""; + private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\."); private Utils() { //no instance @@ -170,6 +172,13 @@ public class Utils { return setsNoPort || usesDefaultPort; } + public static String removeWWWFromUrl(String url) { + if (WWW_PATTERN.matcher(url).find()) { + return url.replace("www.", ""); + } + return url; + } + public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { s = s.substring(1); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index f1a9b35c7..7d5c156b9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -29,9 +29,9 @@ public class SoundcloudParsingHelperTest { } @Test - public void resolveIdWithEmbedPlayerTest() throws Exception { - Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity")); - Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds")); + public void resolveIdWithWidgetApiTest() throws Exception { + Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity")); + Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds")); }