diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ecf831d5d..229555ebb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -16,7 +16,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -41,8 +40,10 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.*; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21 + private static final String HARDCODED_CLIENT_ID = + "TT9Uj7PkasKPYxBlhLNxg2nFm9cLcKmv"; // Updated on 15/05/21 private static String clientId; + public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/"; private SoundcloudParsingHelper() { } @@ -50,7 +51,7 @@ public class SoundcloudParsingHelper { public static synchronized String clientId() throws ExtractionException, IOException { if (!isNullOrEmpty(clientId)) return clientId; - Downloader dl = NewPipe.getDownloader(); + final Downloader dl = NewPipe.getDownloader(); clientId = HARDCODED_CLIENT_ID; if (checkIfHardcodedClientIdIsValid()) { return clientId; @@ -62,20 +63,23 @@ public class SoundcloudParsingHelper { final String responseBody = download.responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; - Document doc = Jsoup.parse(responseBody); - final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]"); + final Document doc = Jsoup.parse(responseBody); + final Elements possibleScripts = doc.select( + "script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]"); // The one containing the client id will likely be the last one Collections.reverse(possibleScripts); final HashMap> headers = new HashMap<>(); headers.put("Range", singletonList("bytes=0-50000")); - for (Element element : possibleScripts) { + for (final Element element : possibleScripts) { final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { - return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody()); - } catch (RegexException ignored) { + clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) + .responseBody()); + return clientId; + } catch (final RegexException ignored) { // Ignore it and proceed to try searching other script } } @@ -85,77 +89,83 @@ public class SoundcloudParsingHelper { throw new ExtractionException("Couldn't extract client id"); } - static boolean checkIfHardcodedClientIdIsValid() { - try { - SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud - .getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); - e.fetchPage(); - return !e.getAudioStreams().isEmpty(); - } catch (Exception ignored) { - // No need to throw an exception here. If something went wrong, the client_id is wrong - return false; - } + static boolean checkIfHardcodedClientIdIsValid() throws IOException, ReCaptchaException { + final int responseCode = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "?client_id=" + + HARDCODED_CLIENT_ID).responseCode(); + // If the response code is 404, it means that the client_id is valid; otherwise, + // it should be not valid + return responseCode == 404; } - public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException { + public static OffsetDateTime parseDateFrom(final String textualUploadDate) + throws ParsingException { try { return OffsetDateTime.parse(textualUploadDate); - } catch (DateTimeParseException e1) { + } catch (final DateTimeParseException e1) { try { - return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000")); - } catch (DateTimeParseException e2) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2); + return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter + .ofPattern("yyyy/MM/dd HH:mm:ss +0000")); + } catch (final DateTimeParseException e2) { + throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + + ", " + e1.getMessage(), e2); } } } /** - * Call the endpoint "/resolve" of the api.

+ * Call the endpoint "/resolve" of the API.

*

* See https://developers.soundcloud.com/docs/api/reference#resolve */ - public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { - String apiUrl = "https://api-v2.soundcloud.com/resolve" - + "?url=" + URLEncoder.encode(url, UTF_8) + public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) + throws IOException, ExtractionException { + final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" + + "?url=" + URLEncoder.encode(url, UTF_8) + "&client_id=" + clientId(); try { - final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody(); + final String response = downloader.get(apiUrl, SoundCloud.getLocalization()) + .responseBody(); return JsonParser.object().from(response); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } } /** - * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api). + * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url + * from the json API). * * @return the url resolved */ - public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException { + public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException, + ReCaptchaException { - String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody(); - return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href"); + return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() + .attr("abs:href"); } /** - * Fetch the widget API with the url and return the id (like the id from the json api). + * Fetch the widget API with the url and return the id (like the id from the json API). * * @return the resolved id */ - public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException { + public static String resolveIdWithWidgetApi(String urlString) throws IOException, + ParsingException { // Remove the tailing slash from URLs due to issues with the SoundCloud API - if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1); - // Make URL lower case and remove www. if it exists. + if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, + urlString.length() - 1); + // Make URL lower case and remove m. and www. if it exists. // Without doing this, the widget API does not recognize the URL. - urlString = Utils.removeWWWFromUrl(urlString.toLowerCase()); + urlString = Utils.removeMAndWWWFromUrl(urlString.toLowerCase()); final URL url; try { url = Utils.stringToURL(urlString); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { throw new IllegalArgumentException("The given URL is not valid"); } @@ -167,22 +177,27 @@ public class SoundcloudParsingHelper { SoundCloud.getLocalization()).responseBody(); final JsonObject o = JsonParser.object().from(response); return String.valueOf(JsonUtils.getValue(o, "id")); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON response", e); - } catch (ExtractionException e) { - throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e); + } catch (final ExtractionException e) { + throw new ParsingException( + "Could not resolve id with embedded player. ClientId not extracted", e); } } /** - * Fetch the users from the given api and commit each of them to the collector. + * Fetch the users from the given API and commit each of them to the collector. *

- * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always - * get MIN_ITEMS or more. + * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense + * that they will always get MIN_ITEMS or more. * - * @param minItems the method will return only when it have extracted that many items (equal or more) + * @param minItems the method will return only when it have extracted that many items + * (equal or more) */ - public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + public static String getUsersFromApiMinItems(final int minItems, + final ChannelInfoItemsCollector collector, + final String apiUrl) throws IOException, + ReCaptchaException, ParsingException { String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -193,23 +208,27 @@ public class SoundcloudParsingHelper { } /** - * Fetch the user items from the given api and commit each of them to the collector. + * Fetch the user items from the given API and commit each of them to the collector. * * @return the next streams url, empty if don't have */ - public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { - String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody(); - JsonObject responseObject; + public static String getUsersFromApi(final ChannelInfoItemsCollector collector, + final String apiUrl) throws IOException, + ReCaptchaException, ParsingException { + final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()) + .responseBody(); + final JsonObject responseObject; + try { responseObject = JsonParser.object().from(response); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } - JsonArray responseCollection = responseObject.getArray("collection"); - for (Object o : responseCollection) { + final JsonArray responseCollection = responseObject.getArray("collection"); + for (final Object o : responseCollection) { if (o instanceof JsonObject) { - JsonObject object = (JsonObject) o; + final JsonObject object = (JsonObject) o; collector.commit(new SoundcloudChannelInfoItemExtractor(object)); } } @@ -217,8 +236,9 @@ public class SoundcloudParsingHelper { String nextPageUrl; try { nextPageUrl = responseObject.getString("next_href"); - if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); - } catch (Exception ignored) { + if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + + SoundcloudParsingHelper.clientId(); + } catch (final Exception ignored) { nextPageUrl = ""; } @@ -226,14 +246,18 @@ public class SoundcloudParsingHelper { } /** - * Fetch the streams from the given api and commit each of them to the collector. + * Fetch the streams from the given API and commit each of them to the collector. *

- * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always - * get MIN_ITEMS or more items. + * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense + * that they will always get MIN_ITEMS or more items. * - * @param minItems the method will return only when it have extracted that many items (equal or more) + * @param minItems the method will return only when it have extracted that many items + * (equal or more) */ - public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + public static String getStreamsFromApiMinItems(final int minItems, + final StreamInfoItemsCollector collector, + final String apiUrl) throws IOException, + ReCaptchaException, ParsingException { String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -244,59 +268,68 @@ public class SoundcloudParsingHelper { } /** - * Fetch the streams from the given api and commit each of them to the collector. + * Fetch the streams from the given API and commit each of them to the collector. * * @return the next streams url, empty if don't have */ - public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { - final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()); + public static String getStreamsFromApi(final StreamInfoItemsCollector collector, + final String apiUrl, + final boolean charts) throws IOException, + ReCaptchaException, ParsingException { + final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud + .getLocalization()); if (response.responseCode() >= 400) { - throw new IOException("Could not get streams from API, HTTP " + response.responseCode()); + throw new IOException("Could not get streams from API, HTTP " + response + .responseCode()); } - JsonObject responseObject; + final JsonObject responseObject; try { responseObject = JsonParser.object().from(response.responseBody()); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } - JsonArray responseCollection = responseObject.getArray("collection"); - for (Object o : responseCollection) { + final JsonArray responseCollection = responseObject.getArray("collection"); + for (final Object o : responseCollection) { if (o instanceof JsonObject) { - JsonObject object = (JsonObject) o; - collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object)); + final JsonObject object = (JsonObject) o; + collector.commit(new SoundcloudStreamInfoItemExtractor(charts + ? object.getObject("track") : object)); } } String nextPageUrl; try { nextPageUrl = responseObject.getString("next_href"); - if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); - } catch (Exception ignored) { + if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + + SoundcloudParsingHelper.clientId(); + } catch (final Exception ignored) { nextPageUrl = ""; } return nextPageUrl; } - public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { + public static String getStreamsFromApi(final StreamInfoItemsCollector collector, + final String apiUrl) throws ReCaptchaException, + ParsingException, IOException { return getStreamsFromApi(collector, apiUrl, false); } @Nonnull - public static String getUploaderUrl(JsonObject object) { - String url = object.getObject("user").getString("permalink_url", EMPTY_STRING); + public static String getUploaderUrl(final JsonObject object) { + final String url = object.getObject("user").getString("permalink_url", EMPTY_STRING); return replaceHttpWithHttps(url); } @Nonnull - public static String getAvatarUrl(JsonObject object) { - String url = object.getObject("user").getString("avatar_url", EMPTY_STRING); + public static String getAvatarUrl(final JsonObject object) { + final String url = object.getObject("user").getString("avatar_url", EMPTY_STRING); return replaceHttpWithHttps(url); } - public static String getUploaderName(JsonObject object) { + public static String getUploaderName(final JsonObject object) { return object.getObject("user").getString("username", EMPTY_STRING); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index e114d030b..f2fc6363f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -23,7 +22,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap public class SoundcloudService extends StreamingService { - public SoundcloudService(int id) { + public SoundcloudService(final int id) { super(id, "SoundCloud", asList(AUDIO, COMMENTS)); } @@ -54,29 +53,29 @@ public class SoundcloudService extends StreamingService { @Override public List getSupportedCountries() { - //Country selector here https://soundcloud.com/charts/top?genre=all-music + // Country selector here: https://soundcloud.com/charts/top?genre=all-music return ContentCountry.listFrom( "AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US" ); } @Override - public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) { - return new SoundcloudStreamExtractor(this, LinkHandler); + public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { + return new SoundcloudStreamExtractor(this, linkHandler); } @Override - public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { + public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { return new SoundcloudChannelExtractor(this, linkHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { return new SoundcloudPlaylistExtractor(this, linkHandler); } @Override - public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { + public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) { return new SoundcloudSearchExtractor(this, queryHandler); } @@ -87,18 +86,11 @@ public class SoundcloudService extends StreamingService { @Override public KioskList getKioskList() throws ExtractionException { - KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, - String url, - String id) - throws ExtractionException { - return new SoundcloudChartsExtractor(SoundcloudService.this, + final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) -> + new SoundcloudChartsExtractor(SoundcloudService.this, new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id); - } - }; - KioskList list = new KioskList(this); + final KioskList list = new KioskList(this); // add kiosks here e.g.: final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory(); @@ -106,7 +98,7 @@ public class SoundcloudService extends StreamingService { list.addKioskEntry(chartsFactory, h, "Top 50"); list.addKioskEntry(chartsFactory, h, "New & hot"); list.setDefaultKiosk("New & hot"); - } catch (Exception e) { + } catch (final Exception e) { throw new ExtractionException(e); } @@ -124,9 +116,8 @@ public class SoundcloudService extends StreamingService { } @Override - public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) + public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) throws ExtractionException { return new SoundcloudCommentsExtractor(this, linkHandler); } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 082de81c0..9191e3961 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -24,22 +25,25 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; private JsonObject user; + private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/"; - public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { + public SoundcloudChannelExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { userId = getLinkHandler().getId(); - final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId + - "?client_id=" + SoundcloudParsingHelper.clientId(); + final String apiUrl = USERS_ENDPOINT + userId + "?client_id=" + + SoundcloudParsingHelper.clientId(); final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody(); try { user = JsonParser.object().from(response); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } } @@ -63,7 +67,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() { - return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url"); + return user.getObject("visuals").getArray("visuals").getObject(0) + .getString("visual_url"); } @Override @@ -105,29 +110,31 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { try { - final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector streamInfoItemsCollector = + new StreamInfoItemsCollector(getServiceId()); - final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=20" - + "&linked_partitioning=1"; + final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id=" + + SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1"; - final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); + final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, + streamInfoItemsCollector, apiUrl); return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl)); - } catch (Exception e) { + } catch (final Exception e) { throw new ExtractionException("Could not get next page", e); } } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl()); + final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, + page.getUrl()); return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java index 39f58d863..b5adfaffd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java @@ -9,7 +9,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject itemObject; - public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) { + public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; } @@ -26,8 +26,8 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getThumbnailUrl() { String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING); - String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg"); - return avatarUrlBetterResolution; + // An avatar URL with a better resolution + return avatarUrl.replace("large.jpg", "crop.jpg"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index f2a88c9fc..5e4ca9b50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -15,17 +15,18 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudChartsExtractor extends KioskExtractor { - public SoundcloudChartsExtractor(StreamingService service, - ListLinkHandler linkHandler, - String kioskId) { + public SoundcloudChartsExtractor(final StreamingService service, + final ListLinkHandler linkHandler, + final String kioskId) { super(service, linkHandler, kioskId); } @Override - public void onFetchPage(@Nonnull Downloader downloader) { + public void onFetchPage(@Nonnull final Downloader downloader) { } @Nonnull @@ -35,13 +36,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true); + final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, + page.getUrl(), true); return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } @@ -51,9 +54,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String apiUrl = "https://api-v2.soundcloud.com/charts" + - "?genre=soundcloud:genres:all-music" + - "&client_id=" + SoundcloudParsingHelper.clientId(); + String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music" + + "&client_id=" + SoundcloudParsingHelper.clientId(); if (getId().equals("Top 50")) { apiUrl += "&kind=top"; @@ -64,15 +66,18 @@ public class SoundcloudChartsExtractor extends KioskExtractor { final ContentCountry contentCountry = SoundCloud.getContentCountry(); String apiUrlWithRegion = null; if (getService().getSupportedCountries().contains(contentCountry)) { - apiUrlWithRegion = apiUrl + "®ion=soundcloud:regions:" + contentCountry.getCountryCode(); + apiUrlWithRegion = apiUrl + "®ion=soundcloud:regions:" + + contentCountry.getCountryCode(); } String nextPageUrl; try { - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true); - } catch (IOException e) { - // Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537 - // we retry without the specified region. + nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, + apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true); + } catch (final IOException e) { + // Request to other region may be geo-restricted. + // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537. + // We retry without the specified region. nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index f12559c3c..b02a3ea80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -24,24 +24,27 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudCommentsExtractor extends CommentsExtractor { - public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { + public SoundcloudCommentsExtractor(final StreamingService service, + final ListLinkHandler uiHandler) { super(service, uiHandler); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + public InfoItemsPage getInitialPage() throws ExtractionException, + IOException { final Downloader downloader = NewPipe.getDownloader(); final Response response = downloader.get(getUrl()); final JsonObject json; try { json = JsonParser.object().from(response.responseBody()); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( + getServiceId()); collectStreamsFrom(collector, json.getArray("collection")); @@ -49,7 +52,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { + public InfoItemsPage getPage(final Page page) throws ExtractionException, + IOException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -60,11 +64,12 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { final JsonObject json; try { json = JsonParser.object().from(response.responseBody()); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( + getServiceId()); collectStreamsFrom(collector, json.getArray("collection")); @@ -74,9 +79,10 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) { } - private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { + private void collectStreamsFrom(final CommentsInfoItemsCollector collector, + final JsonArray entries) throws ParsingException { final String url = getUrl(); - for (Object comment : entries) { + for (final Object comment : entries) { collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index edceb1c86..0eb88f818 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -10,10 +10,10 @@ import javax.annotation.Nullable; import java.util.Objects; public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { - private JsonObject json; - private String url; + private final JsonObject json; + private final String url; - public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) { + public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) { this.json = json; this.url = url; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 4b711e75b..2c4ac60fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -25,6 +25,7 @@ import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @@ -33,22 +34,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private String playlistId; private JsonObject playlist; - public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public SoundcloudPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { playlistId = getLinkHandler().getId(); - String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId + - "?client_id=" + SoundcloudParsingHelper.clientId() + - "&representation=compact"; + final String apiUrl = SOUNDCLOUD_API_V2_URL + "playlists/" + playlistId + "?client_id=" + + SoundcloudParsingHelper.clientId() + "&representation=compact"; - String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody(); + final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody(); try { playlist = JsonParser.object().from(response); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } } @@ -76,11 +78,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { try { final InfoItemsPage infoItems = getInitialPage(); - for (StreamInfoItem item : infoItems.getItems()) { + for (final StreamInfoItem item : infoItems.getItems()) { artworkUrl = item.getThumbnailUrl(); if (!isNullOrEmpty(artworkUrl)) break; } - } catch (Exception ignored) { + } catch (final Exception ignored) { } if (artworkUrl == null) { @@ -139,18 +141,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return ""; } + @Nonnull + @Override public InfoItemsPage getInitialPage() { - final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector streamInfoItemsCollector = + new StreamInfoItemsCollector(getServiceId()); final List ids = new ArrayList<>(); final JsonArray tracks = playlist.getArray("tracks"); - for (Object o : tracks) { + for (final Object o : tracks) { if (o instanceof JsonObject) { final JsonObject track = (JsonObject) o; if (track.has("title")) { // i.e. if full info is available streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); } else { - // %09d would be enough, but a 0 before the number does not create problems, so let's be sure + // %09d would be enough, but a 0 before the number does not create problems, so + // let's be sure ids.add(String.format("%010d", track.getInt("id"))); } } @@ -160,7 +166,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getIds())) { throw new IllegalArgumentException("Page doesn't contain IDs"); } @@ -176,21 +183,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size()); } - final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id=" - + SoundcloudParsingHelper.clientId() - + "&ids=" + Utils.join(",", currentIds); + final String currentPageUrl = SOUNDCLOUD_API_V2_URL + "tracks?client_id=" + + SoundcloudParsingHelper.clientId() + "&ids=" + Utils.join(",", currentIds); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); + final String response = NewPipe.getDownloader().get(currentPageUrl, + getExtractorLocalization()).responseBody(); try { final JsonArray tracks = JsonParser.array().from(response); - for (Object track : tracks) { + for (final Object track : tracks) { if (track instanceof JsonObject) { collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track)); } } - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java index e243780a9..c4b166dba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java @@ -14,7 +14,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr private final JsonObject itemObject; - public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) { + public SoundcloudPlaylistInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; } @@ -34,22 +34,22 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr if (itemObject.isString(ARTWORK_URL_KEY)) { final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { - String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); - return artworkUrlBetterResolution; + // An artwork URL with a better resolution + return artworkUrl.replace("large.jpg", "crop.jpg"); } } try { // Look for artwork url inside the track list - for (Object track : itemObject.getArray("tracks")) { + for (final Object track : itemObject.getArray("tracks")) { final JsonObject trackObject = (JsonObject) track; // First look for track artwork url if (trackObject.isString(ARTWORK_URL_KEY)) { String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { - String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); - return artworkUrlBetterResolution; + // An artwork URL with a better resolution + return artworkUrl.replace("large.jpg", "crop.jpg"); } } @@ -58,14 +58,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING); if (!creatorAvatar.isEmpty()) return creatorAvatar; } - } catch (Exception ignored) { + } catch (final Exception ignored) { // Try other method } try { // Last resort, use user avatar url. If still not found, then throw exception. return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Failed to extract playlist thumbnail url", e); } } @@ -74,7 +74,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr public String getUploaderName() throws ParsingException { try { return itemObject.getObject(USER_KEY).getString("username"); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Failed to extract playlist uploader", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 20d7008ad..eab22da2f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -28,7 +28,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudSearchExtractor extends SearchExtractor { private JsonArray searchCollection; - public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public SoundcloudSearchExtractor(final StreamingService service, + final SearchQueryHandler linkHandler) { super(service, linkHandler); } @@ -52,34 +53,39 @@ public class SoundcloudSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl())); + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl( + getUrl())); } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } final Downloader dl = getDownloader(); try { - final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody(); + final String response = dl.get(page.getUrl(), getExtractorLocalization()) + .responseBody(); searchCollection = JsonParser.object().from(response).getArray("collection"); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } - return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl())); + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page + .getUrl())); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { final Downloader dl = getDownloader(); final String url = getUrl(); try { final String response = dl.get(url, getExtractorLocalization()).responseBody(); searchCollection = JsonParser.object().from(response).getArray("collection"); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } @@ -88,14 +94,14 @@ public class SoundcloudSearchExtractor extends SearchExtractor { } } - private InfoItemsCollector collectItems(JsonArray searchCollection) { + private InfoItemsCollector collectItems( + final JsonArray searchCollection) { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - for (Object result : searchCollection) { + for (final Object result : searchCollection) { if (!(result instanceof JsonObject)) continue; - //noinspection ConstantConditions - JsonObject searchResult = (JsonObject) result; - String kind = searchResult.getString("kind", EMPTY_STRING); + final JsonObject searchResult = (JsonObject) result; + final String kind = searchResult.getString("kind", EMPTY_STRING); switch (kind) { case "user": collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); @@ -112,15 +118,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor { return collector; } - private Page getNextPageFromCurrentUrl(String currentUrl) + private Page getNextPageFromCurrentUrl(final String currentUrl) throws MalformedURLException, UnsupportedEncodingException { final int pageOffset = Integer.parseInt( - Parser.compatParseMap( - new URL(currentUrl) - .getQuery()) - .get("offset")); + Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset")); - return new Page(currentUrl.replace("&offset=" + pageOffset, - "&offset=" + (pageOffset + ITEMS_PER_PAGE))); + return new Page(currentUrl.replace("&offset=" + pageOffset, "&offset=" + + (pageOffset + ITEMS_PER_PAGE))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index c745f1aa7..82bf6706b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -30,18 +30,21 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.*; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; private boolean isAvailable = true; - public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) { + public SoundcloudStreamExtractor(final StreamingService service, + final LinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { track = SoundcloudParsingHelper.resolveFor(downloader, getUrl()); final String policy = track.getString("policy", EMPTY_STRING); @@ -50,9 +53,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { if (policy.equals("SNIP")) { throw new SoundCloudGoPlusContentException(); } - if (policy.equals("BLOCK")) { - throw new GeographicRestrictionException("This track is not available in user's country"); - } + if (policy.equals("BLOCK")) throw new GeographicRestrictionException( + "This track is not available in user's country"); throw new ContentNotAvailableException("Content not available: policy " + policy); } } @@ -80,7 +82,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at"))); + return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString( + "created_at"))); } @Nonnull @@ -220,9 +223,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Nonnull - private static String getTranscodingUrl(final String endpointUrl, final String protocol) throws IOException, ExtractionException { + private static String getTranscodingUrl(final String endpointUrl, + final String protocol) + throws IOException, ExtractionException { final Downloader downloader = NewPipe.getDownloader(); - final String apiStreamUrl = endpointUrl + "?client_id=" + SoundcloudParsingHelper.clientId(); + final String apiStreamUrl = endpointUrl + "?client_id=" + + SoundcloudParsingHelper.clientId(); final String response = downloader.get(apiStreamUrl).responseBody(); final JsonObject urlObject; try { @@ -255,7 +261,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } final String mediaUrl; final String preset = transcodingJsonObject.getString("preset"); - final String protocol = transcodingJsonObject.getObject("format").getString("protocol"); + final String protocol = transcodingJsonObject.getObject("format") + .getString("protocol"); MediaFormat mediaFormat = null; int bitrate = 0; if (preset.contains("mp3")) { @@ -285,7 +292,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } } - /** Parses a SoundCloud HLS manifest to get a single URL of HLS streams. + /** + * Parses a SoundCloud HLS manifest to get a single URL of HLS streams. *

* This method downloads the provided manifest URL, find all web occurrences in the manifest, * get the last segment URL, changes its segment range to {@code 0/track-length} and return @@ -293,7 +301,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { * @param hlsManifestUrl the URL of the manifest to be parsed * @return a single URL that contains a range equal to the length of the track */ - private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException { + private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) + throws ParsingException { final Downloader dl = NewPipe.getDownloader(); final String hlsManifestResponse; @@ -306,11 +315,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { final String[] lines = hlsManifestResponse.split("\\r?\\n"); for (int l = lines.length - 1; l >= 0; l--) { final String line = lines[l]; - // get the last URL from manifest, because it contains the range of the stream + // Get the last URL from manifest, because it contains the range of the stream if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) { final String[] hlsLastRangeUrlArray = line.split("/"); - return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/" - + hlsLastRangeUrlArray[6]; + return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + + "/" + hlsLastRangeUrlArray[6]; } } throw new ParsingException("Could not get any URL from HLS manifest"); @@ -356,7 +365,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId()) + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); @@ -399,15 +408,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - // tags are separated by spaces, but they can be multiple words escaped by quotes " - final String[] tag_list = track.getString("tag_list").split(" "); + // Tags are separated by spaces, but they can be multiple words escaped by quotes " + final String[] tagList = track.getString("tag_list").split(" "); final List tags = new ArrayList<>(); String escapedTag = ""; boolean isEscaped = false; - for (int i = 0; i < tag_list.length; i++) { - String tag = tag_list[i]; + for (int i = 0; i < tagList.length; i++) { + String tag = tagList[i]; if (tag.startsWith("\"")) { - escapedTag += tag_list[i].replace("\"", ""); + escapedTag += tagList[i].replace("\"", ""); isEscaped = true; } else if (isEscaped) { if (tag.endsWith("\"")) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 79d937c2f..90f6efd85 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -14,7 +14,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto protected final JsonObject itemObject; - public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) { + public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java index cabdb453e..14af5ad9a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java @@ -13,12 +13,16 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + /** * Extract the "followings" from a user in SoundCloud. */ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { - public SoundcloudSubscriptionExtractor(SoundcloudService service) { + public SoundcloudSubscriptionExtractor(final SoundcloudService service) { super(service, Collections.singletonList(ContentSource.CHANNEL_URL)); } @@ -28,20 +32,21 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { } @Override - public List fromChannelUrl(String channelUrl) throws IOException, ExtractionException { - if (channelUrl == null) throw new InvalidSourceException("channel url is null"); + public List fromChannelUrl(final String channelUrl) throws IOException, + ExtractionException { + if (channelUrl == null) throw new InvalidSourceException("Channel url is null"); - String id; + final String id; try { id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId(); - } catch (ExtractionException e) { + } catch (final ExtractionException e) { throw new InvalidSourceException(e); } - String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=200"; - ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId()); + final String apiUrl = SOUNDCLOUD_API_V2_URL + "users/" + id + "/followings" + "?client_id=" + + SoundcloudParsingHelper.clientId() + "&limit=200"; + final ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service + .getServiceId()); // ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl); @@ -49,13 +54,13 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { } private String getUrlFrom(String channelUrl) { - channelUrl = channelUrl.replace("http://", "https://").trim(); + channelUrl = replaceHttpWithHttps(channelUrl); - if (!channelUrl.startsWith("https://")) { + if (!channelUrl.startsWith(HTTPS)) { if (!channelUrl.contains("soundcloud.com/")) { channelUrl = "https://soundcloud.com/" + channelUrl; } else { - channelUrl = "https://" + channelUrl; + channelUrl = HTTPS + channelUrl; } } @@ -66,9 +71,9 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { // Utils //////////////////////////////////////////////////////////////////////////*/ - private List toSubscriptionItems(List items) { - List result = new ArrayList<>(items.size()); - for (ChannelInfoItem item : items) { + private List toSubscriptionItems(final List items) { + final List result = new ArrayList<>(items.size()); + for (final ChannelInfoItem item : items) { result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName())); } return result; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java index d8b6a71eb..9b103f073 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java @@ -17,34 +17,34 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; public class SoundcloudSuggestionExtractor extends SuggestionExtractor { - public SoundcloudSuggestionExtractor(StreamingService service) { + public SoundcloudSuggestionExtractor(final StreamingService service) { super(service); } @Override - public List suggestionList(String query) throws IOException, ExtractionException { - List suggestions = new ArrayList<>(); + public List suggestionList(final String query) throws IOException, + ExtractionException { + final List suggestions = new ArrayList<>(); + final Downloader dl = NewPipe.getDownloader(); + final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q=" + + URLEncoder.encode(query, UTF_8) + "&client_id=" + + SoundcloudParsingHelper.clientId() + "&limit=10"; + final String response = dl.get(url, getExtractorLocalization()).responseBody(); - Downloader dl = NewPipe.getDownloader(); - - String url = "https://api-v2.soundcloud.com/search/queries" - + "?q=" + URLEncoder.encode(query, UTF_8) - + "&client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=10"; - - String response = dl.get(url, getExtractorLocalization()).responseBody(); try { - JsonArray collection = JsonParser.object().from(response).getArray("collection"); - for (Object suggestion : collection) { - if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query")); + final JsonArray collection = JsonParser.object().from(response).getArray("collection"); + for (final Object suggestion : collection) { + if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion) + .getString("query")); } return suggestions; - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java index c1308a96f..753fb4139 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java @@ -9,9 +9,10 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.util.List; public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory { - private static final SoundcloudChannelLinkHandlerFactory instance = new SoundcloudChannelLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + - "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; + private static final SoundcloudChannelLinkHandlerFactory instance = + new SoundcloudChannelLinkHandlerFactory(); + private static final String URL_PATTERN ="^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + + "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; public static SoundcloudChannelLinkHandlerFactory getInstance() { return instance; @@ -19,21 +20,24 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) throws ParsingException { try { - return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); - } catch (Exception e) { + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( + "https://api.soundcloud.com/users/" + id); + } catch (final Exception e) { throw new ParsingException(e.getMessage(), e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java index dfe1c80a0..695efee06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java @@ -6,12 +6,13 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.util.List; public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory { - private static final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; - private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; - + private static final String TOP_URL_PATTERN = + "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; + private static final String URL_PATTERN = + "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; @Override - public String getId(String url) { + public String getId(final String url) { if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { return "Top 50"; } else { @@ -20,7 +21,9 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory { } @Override - public String getUrl(String id, List contentFilter, String sortFilter) { + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) { if (id.equals("Top 50")) { return "https://soundcloud.com/charts/top"; } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java index f899c9506..dd5e49ffe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -11,36 +11,40 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory { - private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory(); + private static final SoundcloudCommentsLinkHandlerFactory instance = + new SoundcloudCommentsLinkHandlerFactory(); public static SoundcloudCommentsLinkHandlerFactory getInstance() { return instance; } @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) throws ParsingException { try { - return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() + - "&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new + return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + + clientId() + "&threaded=0" + "&filter_replies=1"; + // Anything but 1 = sort by new // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) - } catch (ExtractionException | IOException e) { + } catch (final ExtractionException | IOException e) { throw new ParsingException("Could not get comments"); } } @Override - public String getId(String url) throws ParsingException { - // delagation to avoid duplicate code, as we need the same id + public String getId(final String url) throws ParsingException { + // Delegation to avoid duplicate code, as we need the same id return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); } @Override - public boolean onAcceptUrl(String url) { + public boolean onAcceptUrl(final String url) { try { getId(url); return true; - } catch (ParsingException e) { + } catch (final ParsingException e) { return false; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java index c89e261c2..ea538bb2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java @@ -9,30 +9,36 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.util.List; public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final SoundcloudPlaylistLinkHandlerFactory instance = new SoundcloudPlaylistLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + - "/sets/[0-9a-z_-]+/?([#?].*)?$"; + private static final SoundcloudPlaylistLinkHandlerFactory instance = + new SoundcloudPlaylistLinkHandlerFactory(); + private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + + "/sets/[0-9a-z_-]+/?([#?].*)?$"; public static SoundcloudPlaylistLinkHandlerFactory getInstance() { return instance; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); - } catch (Exception e) { - throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); + } catch (final Exception e) { + throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), + e); } } @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) + throws ParsingException { try { - return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id); - } catch (Exception e) { + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( + "https://api.soundcloud.com/playlists/" + id); + } catch (final Exception e) { throw new ParsingException(e.getMessage(), e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java index 212f906d7..deb3d8b6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java @@ -11,6 +11,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -23,11 +24,14 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto public static final int ITEMS_PER_PAGE = 10; @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) + throws ParsingException { try { - String url = "https://api-v2.soundcloud.com/search"; + String url = SOUNDCLOUD_API_V2_URL + "search"; - if (contentFilter.size() > 0) { + if (!contentFilter.isEmpty()) { switch (contentFilter.get(0)) { case TRACKS: url += "/tracks"; @@ -44,16 +48,15 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto } } - return url + "?q=" + URLEncoder.encode(id, UTF_8) - + "&client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=" + ITEMS_PER_PAGE + return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id=" + + SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE + "&offset=0"; - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); - } catch (ReCaptchaException e) { + } catch (final ReCaptchaException e) { throw new ParsingException("ReCaptcha required", e); - } catch (IOException | ExtractionException e) { + } catch (final IOException | ExtractionException e) { throw new ParsingException("Could not get client id", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 55eafa8a3..cda95976b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -7,9 +7,10 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { - private static final SoundcloudStreamLinkHandlerFactory instance = new SoundcloudStreamLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + - "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; + private static final SoundcloudStreamLinkHandlerFactory instance = + new SoundcloudStreamLinkHandlerFactory(); + private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; private SoundcloudStreamLinkHandlerFactory() { } @@ -19,21 +20,22 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { } @Override - public String getUrl(String id) throws ParsingException { + public String getUrl(final String id) throws ParsingException { try { - return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id); - } catch (Exception e) { + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( + "https://api.soundcloud.com/tracks/" + id); + } catch (final Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException(e.getMessage(), e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index baf502ea6..d47d9afb6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -15,10 +15,11 @@ public class Utils { public static final String HTTPS = "https://"; public static final String UTF_8 = "UTF-8"; public static final String EMPTY_STRING = ""; + private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\."); private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\."); private Utils() { - //no instance + // no instance } /** @@ -30,7 +31,7 @@ public class Utils { * @param toRemove string to remove non-digit chars * @return a string that contains only digits */ - public static String removeNonDigitCharacters(String toRemove) { + public static String removeNonDigitCharacters(final String toRemove) { return toRemove.replaceAll("\\D+", ""); } @@ -48,7 +49,8 @@ public class Utils { * @throws NumberFormatException * @throws ParsingException */ - public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException { + public static long mixedNumberWordToLong(final String numberWord) throws NumberFormatException, + ParsingException { String multiplier = ""; try { multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2); @@ -74,7 +76,7 @@ public class Utils { * @param pattern the pattern that will be used to check the url * @param url the url to be tested */ - public static void checkUrl(String pattern, String url) throws ParsingException { + public static void checkUrl(final String pattern, final String url) throws ParsingException { if (isNullOrEmpty(url)) { throw new IllegalArgumentException("Url can't be null or empty"); } @@ -101,14 +103,14 @@ public class Utils { } /** - * get the value of a URL-query by name. - * if a url-query is give multiple times, only the value of the first query is returned + * Get the value of a URL-query by name. + * If a url-query is give multiple times, only the value of the first query is returned * * @param url the url to be used * @param parameterName the pattern that will be used to check the url * @return a string that contains the value of the query parameter or null if nothing was found */ - public static String getQueryValue(URL url, String parameterName) { + public static String getQueryValue(final URL url, final String parameterName) { String urlQuery = url.getQuery(); if (urlQuery != null) { @@ -118,8 +120,9 @@ public class Utils { String query; try { query = URLDecoder.decode(params[0], UTF_8); - } catch (UnsupportedEncodingException e) { - System.err.println("Cannot decode string with UTF-8. using the string without decoding"); + } catch (final UnsupportedEncodingException e) { + System.err.println( + "Cannot decode string with UTF-8. using the string without decoding"); e.printStackTrace(); query = params[0]; } @@ -127,8 +130,9 @@ public class Utils { if (query.equals(parameterName)) { try { return URLDecoder.decode(params[1], UTF_8); - } catch (UnsupportedEncodingException e) { - System.err.println("Cannot decode string with UTF-8. using the string without decoding"); + } catch (final UnsupportedEncodingException e) { + System.err.println( + "Cannot decode string with UTF-8. using the string without decoding"); e.printStackTrace(); return params[1]; } @@ -146,7 +150,7 @@ public class Utils { * @param url the string to be converted to a URL-Object * @return a URL-Object containing the url */ - public static URL stringToURL(String url) throws MalformedURLException { + public static URL stringToURL(final String url) throws MalformedURLException { try { return new URL(url); } catch (MalformedURLException e) { @@ -159,7 +163,7 @@ public class Utils { } } - public static boolean isHTTP(URL url) { + public static boolean isHTTP(final URL url) { // make sure its http or https String protocol = url.getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) { @@ -172,7 +176,10 @@ public class Utils { return setsNoPort || usesDefaultPort; } - public static String removeWWWFromUrl(String url) { + public static String removeMAndWWWFromUrl(final String url) { + if (M_PATTERN.matcher(url).find()) { + return url.replace("m.", ""); + } if (WWW_PATTERN.matcher(url).find()) { return url.replace("www.", ""); } @@ -216,7 +223,8 @@ public class Utils { try { final URL decoded = Utils.stringToURL(url); if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { - return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8); + return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), + UTF_8); } } catch (final Exception ignored) { } @@ -258,7 +266,8 @@ public class Utils { return true; } - public static String join(final CharSequence delimiter, final Iterable elements) { + public static String join(final CharSequence delimiter, + final Iterable elements) { final StringBuilder stringBuilder = new StringBuilder(); final Iterator iterator = elements.iterator(); while (iterator.hasNext()) { @@ -283,7 +292,8 @@ public class Utils { /** * Concatenate all non-null, non-empty and strings which are not equal to "null". */ - public static String nonEmptyAndNullJoin(final CharSequence delimiter, final String[] elements) { + public static String nonEmptyAndNullJoin(final CharSequence delimiter, + final String[] elements) { final List list = new java.util.ArrayList<>(Arrays.asList(elements)); list.removeIf(s -> isNullOrEmpty(s) || s.equals("null")); return join(delimiter, list);