be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #627 from TiA4f8R/use-snd-api-v2-everywhere 

[SoundCloud] Use a lightweight request to check if the hardcoded client_id is valid, fix the extraction of mobile URLs and more
This commit is contained in:
Tobi 2021-05-23 22:52:40 +02:00 committed by GitHub
parent 636e27333b a00fdcbd3d
commit ff005122bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 407 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -16,7 +16,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -41,8 +40,10 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21
private static final String HARDCODED_CLIENT_ID =
"TT9Uj7PkasKPYxBlhLNxg2nFm9cLcKmv"; // Updated on 15/05/21
private static String clientId;
public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
private SoundcloudParsingHelper() {
}
@ -50,7 +51,7 @@ public class SoundcloudParsingHelper {
public static synchronized String clientId() throws ExtractionException, IOException {
if (!isNullOrEmpty(clientId)) return clientId;
Downloader dl = NewPipe.getDownloader();
final Downloader dl = NewPipe.getDownloader();
clientId = HARDCODED_CLIENT_ID;
if (checkIfHardcodedClientIdIsValid()) {
return clientId;
@ -62,20 +63,23 @@ public class SoundcloudParsingHelper {
final String responseBody = download.responseBody();
final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
final Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select(
"script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-50000"));
for (Element element : possibleScripts) {
for (final Element element : possibleScripts) {
final String srcUrl = element.attr("src");
if (!isNullOrEmpty(srcUrl)) {
try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody());
} catch (RegexException ignored) {
clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers)
.responseBody());
return clientId;
} catch (final RegexException ignored) {
// Ignore it and proceed to try searching other script
}
}
@ -85,77 +89,83 @@ public class SoundcloudParsingHelper {
throw new ExtractionException("Couldn't extract client id");
}
static boolean checkIfHardcodedClientIdIsValid() {
try {
SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud
.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
e.fetchPage();
return !e.getAudioStreams().isEmpty();
} catch (Exception ignored) {
// No need to throw an exception here. If something went wrong, the client_id is wrong
return false;
}
static boolean checkIfHardcodedClientIdIsValid() throws IOException, ReCaptchaException {
final int responseCode = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "?client_id="
+ HARDCODED_CLIENT_ID).responseCode();
// If the response code is 404, it means that the client_id is valid; otherwise,
// it should be not valid
return responseCode == 404;
}
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
public static OffsetDateTime parseDateFrom(final String textualUploadDate)
throws ParsingException {
try {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) {
} catch (final DateTimeParseException e1) {
try {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter
.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (final DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\""
+ ", " + e1.getMessage(), e2);
}
}
}
/**
* Call the endpoint "/resolve" of the api.<p>
* Call the endpoint "/resolve" of the API.<p>
* <p>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve"
public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
throws IOException, ExtractionException {
final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId();
try {
final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody();
final String response = downloader.get(apiUrl, SoundCloud.getLocalization())
.responseBody();
return JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
/**
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url
* from the json API).
*
* @return the url resolved
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException {
public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
ReCaptchaException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
}
/**
* Fetch the widget API with the url and return the id (like the id from the json api).
* Fetch the widget API with the url and return the id (like the id from the json API).
*
* @return the resolved id
*/
public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
public static String resolveIdWithWidgetApi(String urlString) throws IOException,
ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
// Make URL lower case and remove www. if it exists.
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0,
urlString.length() - 1);
// Make URL lower case and remove m. and www. if it exists.
// Without doing this, the widget API does not recognize the URL.
urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
urlString = Utils.removeMAndWWWFromUrl(urlString.toLowerCase());
final URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
@ -167,22 +177,27 @@ public class SoundcloudParsingHelper {
SoundCloud.getLocalization()).responseBody();
final JsonObject o = JsonParser.object().from(response);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e);
} catch (ExtractionException e) {
throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
} catch (final ExtractionException e) {
throw new ParsingException(
"Could not resolve id with embedded player. ClientId not extracted", e);
}
}
/**
* Fetch the users from the given api and commit each of them to the collector.
* Fetch the users from the given API and commit each of them to the collector.
* <p>
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more.
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
* @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getUsersFromApiMinItems(final int minItems,
final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -193,23 +208,27 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the user items from the given api and commit each of them to the collector.
* Fetch the user items from the given API and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization())
.responseBody();
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
final JsonArray responseCollection = responseObject.getArray("collection");
for (final Object o : responseCollection) {
if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o;
final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudChannelInfoItemExtractor(object));
}
}
@ -217,8 +236,9 @@ public class SoundcloudParsingHelper {
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
+ SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = "";
}
@ -226,14 +246,18 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* Fetch the streams from the given API and commit each of them to the collector.
* <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more items.
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more items.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
* @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getStreamsFromApiMinItems(final int minItems,
final StreamInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -244,59 +268,68 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* Fetch the streams from the given API and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
final boolean charts) throws IOException,
ReCaptchaException, ParsingException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud
.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
throw new IOException("Could not get streams from API, HTTP " + response
.responseCode());
}
JsonObject responseObject;
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
final JsonArray responseCollection = responseObject.getArray("collection");
for (final Object o : responseCollection) {
if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object));
final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudStreamInfoItemExtractor(charts
? object.getObject("track") : object));
}
}
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
+ SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = "";
}
return nextPageUrl;
}
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl) throws ReCaptchaException,
ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false);
}
@Nonnull
public static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
public static String getUploaderUrl(final JsonObject object) {
final String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
@Nonnull
public static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
public static String getAvatarUrl(final JsonObject object) {
final String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
public static String getUploaderName(JsonObject object) {
public static String getUploaderName(final JsonObject object) {
return object.getObject("user").getString("username", EMPTY_STRING);
}
}

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,7 +22,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
public SoundcloudService(final int id) {
super(id, "SoundCloud", asList(AUDIO, COMMENTS));
}
@ -54,29 +53,29 @@ public class SoundcloudService extends StreamingService {
@Override
public List<ContentCountry> getSupportedCountries() {
//Country selector here https://soundcloud.com/charts/top?genre=all-music
// Country selector here: https://soundcloud.com/charts/top?genre=all-music
return ContentCountry.listFrom(
"AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US"
);
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
return new SoundcloudStreamExtractor(this, LinkHandler);
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new SoundcloudStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
return new SoundcloudSearchExtractor(this, queryHandler);
}
@ -87,18 +86,11 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
}
};
KioskList list = new KioskList(this);
final KioskList list = new KioskList(this);
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
@ -106,7 +98,7 @@ public class SoundcloudService extends StreamingService {
list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot");
list.setDefaultKiosk("New & hot");
} catch (Exception e) {
} catch (final Exception e) {
throw new ExtractionException(e);
}
@ -124,9 +116,8 @@ public class SoundcloudService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
return new SoundcloudCommentsExtractor(this, linkHandler);
}
}

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -24,22 +25,25 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JsonObject user;
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
public SoundcloudChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
userId = getLinkHandler().getId();
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
final String apiUrl = USERS_ENDPOINT + userId + "?client_id="
+ SoundcloudParsingHelper.clientId();
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@ -63,7 +67,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() {
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
return user.getObject("visuals").getArray("visuals").getObject(0)
.getString("visual_url");
}
@Override
@ -105,29 +110,31 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
try {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (Exception e) {
} catch (final Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -9,7 +9,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final JsonObject itemObject;
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}
@ -26,8 +26,8 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
// An avatar URL with a better resolution
return avatarUrl.replace("large.jpg", "crop.jpg");
}
@Override

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
View File

@ -15,17 +15,18 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId) {
public SoundcloudChartsExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final String kioskId) {
super(service, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) {
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@ -35,13 +36,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true);
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
page.getUrl(), true);
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
@ -51,9 +54,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
"&client_id=" + SoundcloudParsingHelper.clientId();
String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
+ "&client_id=" + SoundcloudParsingHelper.clientId();
if (getId().equals("Top 50")) {
apiUrl += "&kind=top";
@ -64,15 +66,18 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
final ContentCountry contentCountry = SoundCloud.getContentCountry();
String apiUrlWithRegion = null;
if (getService().getSupportedCountries().contains(contentCountry)) {
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
+ contentCountry.getCountryCode();
}
String nextPageUrl;
try {
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (IOException e) {
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
// we retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (final IOException e) {
// Request to other region may be geo-restricted.
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
// We retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -24,24 +24,27 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor {
public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
public SoundcloudCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException, IOException {
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
@ -49,7 +52,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException, IOException {
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
IOException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -60,11 +64,12 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
@ -74,9 +79,10 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException {
private void collectStreamsFrom(final CommentsInfoItemsCollector collector,
final JsonArray entries) throws ParsingException {
final String url = getUrl();
for (Object comment : entries) {
for (final Object comment : entries) {
collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -10,10 +10,10 @@ import javax.annotation.Nullable;
import java.util.Objects;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private JsonObject json;
private String url;
private final JsonObject json;
private final String url;
public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) {
public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) {
this.json = json;
this.url = url;
}

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@ -33,22 +34,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
public SoundcloudPlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
playlistId = getLinkHandler().getId();
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
final String apiUrl = SOUNDCLOUD_API_V2_URL + "playlists/" + playlistId + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&representation=compact";
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@ -76,11 +78,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
try {
final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
for (StreamInfoItem item : infoItems.getItems()) {
for (final StreamInfoItem item : infoItems.getItems()) {
artworkUrl = item.getThumbnailUrl();
if (!isNullOrEmpty(artworkUrl)) break;
}
} catch (Exception ignored) {
} catch (final Exception ignored) {
}
if (artworkUrl == null) {
@ -139,18 +141,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final List<String> ids = new ArrayList<>();
final JsonArray tracks = playlist.getArray("tracks");
for (Object o : tracks) {
for (final Object o : tracks) {
if (o instanceof JsonObject) {
final JsonObject track = (JsonObject) o;
if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else {
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure
// %09d would be enough, but a 0 before the number does not create problems, so
// let's be sure
ids.add(String.format("%010d", track.getInt("id")));
}
}
@ -160,7 +166,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getIds())) {
throw new IllegalArgumentException("Page doesn't contain IDs");
}
@ -176,21 +183,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size());
}
final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id="
+ SoundcloudParsingHelper.clientId()
+ "&ids=" + Utils.join(",", currentIds);
final String currentPageUrl = SOUNDCLOUD_API_V2_URL + "tracks?client_id="
+ SoundcloudParsingHelper.clientId() + "&ids=" + Utils.join(",", currentIds);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
final String response = NewPipe.getDownloader().get(currentPageUrl,
getExtractorLocalization()).responseBody();
try {
final JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) {
for (final Object track : tracks) {
if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
}
}
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -14,7 +14,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
private final JsonObject itemObject;
public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
public SoundcloudPlaylistInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}
@ -34,22 +34,22 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
// An artwork URL with a better resolution
return artworkUrl.replace("large.jpg", "crop.jpg");
}
}
try {
// Look for artwork url inside the track list
for (Object track : itemObject.getArray("tracks")) {
for (final Object track : itemObject.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
// An artwork URL with a better resolution
return artworkUrl.replace("large.jpg", "crop.jpg");
}
}
@ -58,14 +58,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
} catch (final Exception ignored) {
// Try other method
}
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
}
@ -74,7 +74,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
public String getUploaderName() throws ParsingException {
try {
return itemObject.getObject(USER_KEY).getString("username");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -28,7 +28,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@ -52,34 +53,39 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl()));
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(
getUrl()));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Downloader dl = getDownloader();
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody();
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl()));
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page
.getUrl()));
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final Downloader dl = getDownloader();
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -88,14 +94,14 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
}
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) {
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(
final JsonArray searchCollection) {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
for (Object result : searchCollection) {
for (final Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", EMPTY_STRING);
final JsonObject searchResult = (JsonObject) result;
final String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
@ -112,15 +118,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return collector;
}
private Page getNextPageFromCurrentUrl(String currentUrl)
private Page getNextPageFromCurrentUrl(final String currentUrl)
throws MalformedURLException, UnsupportedEncodingException {
final int pageOffset = Integer.parseInt(
Parser.compatParseMap(
new URL(currentUrl)
.getQuery())
.get("offset"));
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
return new Page(currentUrl.replace("&offset=" + pageOffset,
"&offset=" + (pageOffset + ITEMS_PER_PAGE)));
return new Page(currentUrl.replace("&offset=" + pageOffset, "&offset="
+ (pageOffset + ITEMS_PER_PAGE)));
}
}

49
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -30,18 +30,21 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
private boolean isAvailable = true;
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
public SoundcloudStreamExtractor(final StreamingService service,
final LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
final String policy = track.getString("policy", EMPTY_STRING);
@ -50,9 +53,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
if (policy.equals("SNIP")) {
throw new SoundCloudGoPlusContentException();
}
if (policy.equals("BLOCK")) {
throw new GeographicRestrictionException("This track is not available in user's country");
}
if (policy.equals("BLOCK")) throw new GeographicRestrictionException(
"This track is not available in user's country");
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
}
@ -80,7 +82,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at")));
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString(
"created_at")));
}
@Nonnull
@ -220,9 +223,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Nonnull
private static String getTranscodingUrl(final String endpointUrl, final String protocol) throws IOException, ExtractionException {
private static String getTranscodingUrl(final String endpointUrl,
final String protocol)
throws IOException, ExtractionException {
final Downloader downloader = NewPipe.getDownloader();
final String apiStreamUrl = endpointUrl + "?client_id=" + SoundcloudParsingHelper.clientId();
final String apiStreamUrl = endpointUrl + "?client_id="
+ SoundcloudParsingHelper.clientId();
final String response = downloader.get(apiStreamUrl).responseBody();
final JsonObject urlObject;
try {
@ -255,7 +261,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
final String mediaUrl;
final String preset = transcodingJsonObject.getString("preset");
final String protocol = transcodingJsonObject.getObject("format").getString("protocol");
final String protocol = transcodingJsonObject.getObject("format")
.getString("protocol");
MediaFormat mediaFormat = null;
int bitrate = 0;
if (preset.contains("mp3")) {
@ -285,7 +292,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
}
/** Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
/**
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
* <p>
* This method downloads the provided manifest URL, find all web occurrences in the manifest,
* get the last segment URL, changes its segment range to {@code 0/track-length} and return
@ -293,7 +301,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
* @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track
*/
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException {
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
throws ParsingException {
final Downloader dl = NewPipe.getDownloader();
final String hlsManifestResponse;
@ -306,11 +315,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
final String[] lines = hlsManifestResponse.split("\\r?\\n");
for (int l = lines.length - 1; l >= 0; l--) {
final String line = lines[l];
// get the last URL from manifest, because it contains the range of the stream
// Get the last URL from manifest, because it contains the range of the stream
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
final String[] hlsLastRangeUrlArray = line.split("/");
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/"
+ hlsLastRangeUrlArray[6];
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
+ "/" + hlsLastRangeUrlArray[6];
}
}
throw new ParsingException("Could not get any URL from HLS manifest");
@ -356,7 +365,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
@ -399,15 +408,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
// tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tag_list = track.getString("tag_list").split(" ");
// Tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tagList = track.getString("tag_list").split(" ");
final List<String> tags = new ArrayList<>();
String escapedTag = "";
boolean isEscaped = false;
for (int i = 0; i < tag_list.length; i++) {
String tag = tag_list[i];
for (int i = 0; i < tagList.length; i++) {
String tag = tagList[i];
if (tag.startsWith("\"")) {
escapedTag += tag_list[i].replace("\"", "");
escapedTag += tagList[i].replace("\"", "");
isEscaped = true;
} else if (isEscaped) {
if (tag.endsWith("\"")) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -14,7 +14,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
protected final JsonObject itemObject;
public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}

35
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java
View File

@ -13,12 +13,16 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
/**
* Extract the "followings" from a user in SoundCloud.
*/
public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
public SoundcloudSubscriptionExtractor(SoundcloudService service) {
public SoundcloudSubscriptionExtractor(final SoundcloudService service) {
super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
}
@ -28,20 +32,21 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
}
@Override
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("channel url is null");
public List<SubscriptionItem> fromChannelUrl(final String channelUrl) throws IOException,
ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("Channel url is null");
String id;
final String id;
try {
id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId();
} catch (ExtractionException e) {
} catch (final ExtractionException e) {
throw new InvalidSourceException(e);
}
String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=200";
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());
final String apiUrl = SOUNDCLOUD_API_V2_URL + "users/" + id + "/followings" + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=200";
final ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service
.getServiceId());
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
@ -49,13 +54,13 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
}
private String getUrlFrom(String channelUrl) {
channelUrl = channelUrl.replace("http://", "https://").trim();
channelUrl = replaceHttpWithHttps(channelUrl);
if (!channelUrl.startsWith("https://")) {
if (!channelUrl.startsWith(HTTPS)) {
if (!channelUrl.contains("soundcloud.com/")) {
channelUrl = "https://soundcloud.com/" + channelUrl;
} else {
channelUrl = "https://" + channelUrl;
channelUrl = HTTPS + channelUrl;
}
}
@ -66,9 +71,9 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
// Utils
//////////////////////////////////////////////////////////////////////////*/
private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) {
List<SubscriptionItem> result = new ArrayList<>(items.size());
for (ChannelInfoItem item : items) {
private List<SubscriptionItem> toSubscriptionItems(final List<ChannelInfoItem> items) {
final List<SubscriptionItem> result = new ArrayList<>(items.size());
for (final ChannelInfoItem item : items) {
result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
}
return result;

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View File

@ -17,34 +17,34 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) {
public SoundcloudSuggestionExtractor(final StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
List<String> suggestions = new ArrayList<>();
public List<String> suggestionList(final String query) throws IOException,
ExtractionException {
final List<String> suggestions = new ArrayList<>();
final Downloader dl = NewPipe.getDownloader();
final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q="
+ URLEncoder.encode(query, UTF_8) + "&client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=10";
final String response = dl.get(url, getExtractorLocalization()).responseBody();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.get(url, getExtractorLocalization()).responseBody();
try {
JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query"));
final JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (final Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion)
.getString("query"));
}
return suggestions;
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java
View File

@ -9,9 +9,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List;
public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChannelLinkHandlerFactory instance = new SoundcloudChannelLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
private static final SoundcloudChannelLinkHandlerFactory instance =
new SoundcloudChannelLinkHandlerFactory();
private static final String URL_PATTERN ="^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
public static SoundcloudChannelLinkHandlerFactory getInstance() {
return instance;
@ -19,21 +20,24 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/users/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java
View File

@ -6,12 +6,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
private static final String TOP_URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
@Override
public String getId(String url) {
public String getId(final String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
@ -20,7 +21,9 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java
View File

@ -11,36 +11,40 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing
public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory();
private static final SoundcloudCommentsLinkHandlerFactory instance =
new SoundcloudCommentsLinkHandlerFactory();
public static SoundcloudCommentsLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() +
"&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id="
+ clientId() + "&threaded=0" + "&filter_replies=1";
// Anything but 1 = sort by new
// + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10)
// + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl)
} catch (ExtractionException | IOException e) {
} catch (final ExtractionException | IOException e) {
throw new ParsingException("Could not get comments");
}
}
@Override
public String getId(String url) throws ParsingException {
// delagation to avoid duplicate code, as we need the same id
public String getId(final String url) throws ParsingException {
// Delegation to avoid duplicate code, as we need the same id
return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url);
}
@Override
public boolean onAcceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java
View File

@ -9,30 +9,36 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List;
public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudPlaylistLinkHandlerFactory instance = new SoundcloudPlaylistLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/sets/[0-9a-z_-]+/?([#?].*)?$";
private static final SoundcloudPlaylistLinkHandlerFactory instance =
new SoundcloudPlaylistLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/sets/[0-9a-z_-]+/?([#?].*)?$";
public static SoundcloudPlaylistLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
} catch (final Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(),
e);
}
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/playlists/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View File

@ -11,6 +11,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -23,11 +24,14 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
public static final int ITEMS_PER_PAGE = 10;
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try {
String url = "https://api-v2.soundcloud.com/search";
String url = SOUNDCLOUD_API_V2_URL + "search";
if (contentFilter.size() > 0) {
if (!contentFilter.isEmpty()) {
switch (contentFilter.get(0)) {
case TRACKS:
url += "/tracks";
@ -44,16 +48,15 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
}
}
return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE
return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE
+ "&offset=0";
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
} catch (ReCaptchaException e) {
} catch (final ReCaptchaException e) {
throw new ParsingException("ReCaptcha required", e);
} catch (IOException | ExtractionException e) {
} catch (final IOException | ExtractionException e) {
throw new ParsingException("Could not get client id", e);
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java
View File

@ -7,9 +7,10 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final SoundcloudStreamLinkHandlerFactory instance = new SoundcloudStreamLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private static final SoundcloudStreamLinkHandlerFactory instance =
new SoundcloudStreamLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private SoundcloudStreamLinkHandlerFactory() {
}
@ -19,21 +20,22 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(String id) throws ParsingException {
public String getUrl(final String id) throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/tracks/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -15,10 +15,11 @@ public class Utils {
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
private Utils() {
//no instance
// no instance
}
/**
@ -30,7 +31,7 @@ public class Utils {
* @param toRemove string to remove non-digit chars
* @return a string that contains only digits
*/
public static String removeNonDigitCharacters(String toRemove) {
public static String removeNonDigitCharacters(final String toRemove) {
return toRemove.replaceAll("\\D+", "");
}
@ -48,7 +49,8 @@ public class Utils {
* @throws NumberFormatException
* @throws ParsingException
*/
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
public static long mixedNumberWordToLong(final String numberWord) throws NumberFormatException,
ParsingException {
String multiplier = "";
try {
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
@ -74,7 +76,7 @@ public class Utils {
* @param pattern the pattern that will be used to check the url
* @param url the url to be tested
*/
public static void checkUrl(String pattern, String url) throws ParsingException {
public static void checkUrl(final String pattern, final String url) throws ParsingException {
if (isNullOrEmpty(url)) {
throw new IllegalArgumentException("Url can't be null or empty");
}
@ -101,14 +103,14 @@ public class Utils {
}
/**
* get the value of a URL-query by name.
* if a url-query is give multiple times, only the value of the first query is returned
* Get the value of a URL-query by name.
* If a url-query is give multiple times, only the value of the first query is returned
*
* @param url the url to be used
* @param parameterName the pattern that will be used to check the url
* @return a string that contains the value of the query parameter or null if nothing was found
*/
public static String getQueryValue(URL url, String parameterName) {
public static String getQueryValue(final URL url, final String parameterName) {
String urlQuery = url.getQuery();
if (urlQuery != null) {
@ -118,8 +120,9 @@ public class Utils {
String query;
try {
query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
} catch (final UnsupportedEncodingException e) {
System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
query = params[0];
}
@ -127,8 +130,9 @@ public class Utils {
if (query.equals(parameterName)) {
try {
return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
} catch (final UnsupportedEncodingException e) {
System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
return params[1];
}
@ -146,7 +150,7 @@ public class Utils {
* @param url the string to be converted to a URL-Object
* @return a URL-Object containing the url
*/
public static URL stringToURL(String url) throws MalformedURLException {
public static URL stringToURL(final String url) throws MalformedURLException {
try {
return new URL(url);
} catch (MalformedURLException e) {
@ -159,7 +163,7 @@ public class Utils {
}
}
public static boolean isHTTP(URL url) {
public static boolean isHTTP(final URL url) {
// make sure its http or https
String protocol = url.getProtocol();
if (!protocol.equals("http") && !protocol.equals("https")) {
@ -172,7 +176,10 @@ public class Utils {
return setsNoPort || usesDefaultPort;
}
public static String removeWWWFromUrl(String url) {
public static String removeMAndWWWFromUrl(final String url) {
if (M_PATTERN.matcher(url).find()) {
return url.replace("m.", "");
}
if (WWW_PATTERN.matcher(url).find()) {
return url.replace("www.", "");
}
@ -216,7 +223,8 @@ public class Utils {
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url),
UTF_8);
}
} catch (final Exception ignored) {
}
@ -258,7 +266,8 @@ public class Utils {
return true;
}
public static String join(final CharSequence delimiter, final Iterable<? extends CharSequence> elements) {
public static String join(final CharSequence delimiter,
final Iterable<? extends CharSequence> elements) {
final StringBuilder stringBuilder = new StringBuilder();
final Iterator<? extends CharSequence> iterator = elements.iterator();
while (iterator.hasNext()) {
@ -283,7 +292,8 @@ public class Utils {
/**
* Concatenate all non-null, non-empty and strings which are not equal to <code>"null"</code>.
*/
public static String nonEmptyAndNullJoin(final CharSequence delimiter, final String[] elements) {
public static String nonEmptyAndNullJoin(final CharSequence delimiter,
final String[] elements) {
final List<String> list = new java.util.ArrayList<>(Arrays.asList(elements));
list.removeIf(s -> isNullOrEmpty(s) || s.equals("null"));
return join(delimiter, list);