diff --git a/MediaFormat.java b/MediaFormat.java index 88f41e5d4..9c2ed4c43 100644 --- a/MediaFormat.java +++ b/MediaFormat.java @@ -34,7 +34,8 @@ public enum MediaFormat { WEBM (0x2, "WebM", "webm", "video/webm"), // audio formats M4A (0x3, "m4a", "m4a", "audio/mp4"), - WEBMA (0x4, "WebM", "webm", "audio/webm"); + WEBMA (0x4, "WebM", "webm", "audio/webm"), + MP3 (0x5, "MP3", "mp3", "audio/mpeg"); public final int id; @SuppressWarnings("WeakerAccess") diff --git a/ServiceList.java b/ServiceList.java index e92b8321e..26a748368 100644 --- a/ServiceList.java +++ b/ServiceList.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; /* * Created by the-scrabi on 18.02.17. @@ -8,6 +9,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeService; class ServiceList { public static final StreamingService[] serviceList = { - new YoutubeService(0) + new YoutubeService(0), + new SoundcloudService(1) }; } diff --git a/UrlIdHandler.java b/UrlIdHandler.java index 24da9cc6f..9e5739e33 100644 --- a/UrlIdHandler.java +++ b/UrlIdHandler.java @@ -1,6 +1,9 @@ package org.schabi.newpipe.extractor; +import java.io.IOException; + import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; /* * Created by Christian Schabesberger on 26.07.16. @@ -24,7 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; public interface UrlIdHandler { - String getUrl(String videoId); + String getUrl(String videoId) throws ParsingException; String getId(String siteUrl) throws ParsingException; String cleanUrl(String siteUrl) throws ParsingException; diff --git a/channel/ChannelExtractor.java b/channel/ChannelExtractor.java index 1ce8dc638..64311ef06 100644 --- a/channel/ChannelExtractor.java +++ b/channel/ChannelExtractor.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; @@ -39,7 +40,7 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getAvatarUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; - public abstract StreamInfoItemCollector getStreams() throws ParsingException; + public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException; public abstract long getSubscriberCount() throws ParsingException; } diff --git a/playlist/PlaylistExtractor.java b/playlist/PlaylistExtractor.java index 97511b5d6..514c10ece 100644 --- a/playlist/PlaylistExtractor.java +++ b/playlist/PlaylistExtractor.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; @@ -21,6 +22,6 @@ public abstract class PlaylistExtractor extends ListExtractor { public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderName() throws ParsingException; public abstract String getUploaderAvatarUrl() throws ParsingException; - public abstract StreamInfoItemCollector getStreams() throws ParsingException; + public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException; public abstract long getStreamsCount() throws ParsingException; } diff --git a/services/soundcloud/SoundcloudChannelExtractor.java b/services/soundcloud/SoundcloudChannelExtractor.java new file mode 100644 index 000000000..7c1f0be6e --- /dev/null +++ b/services/soundcloud/SoundcloudChannelExtractor.java @@ -0,0 +1,118 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +@SuppressWarnings("WeakerAccess") +public class SoundcloudChannelExtractor extends ChannelExtractor { + private String channelId; + private JSONObject channel; + private String nextUrl; + + public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { + super(urlIdHandler, url, serviceId); + + Downloader dl = NewPipe.getDownloader(); + + channelId = urlIdHandler.getId(url); + String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + channel = new JSONObject(response); + } + + @Override + public String getChannelId() { + return channelId; + } + + @Override + public String getChannelName() { + return channel.getString("username"); + } + + @Override + public String getAvatarUrl() { + return channel.getString("avatar_url"); + } + + @Override + public String getBannerUrl() throws ParsingException { + try { + return channel.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url"); + } catch (Exception e) { + throw new ParsingException("Could not get Banner", e); + } + } + + @Override + public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException { + StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + Downloader dl = NewPipe.getDownloader(); + + String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks" + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=10" + + "&offset=0" + + "&linked_partitioning=1"; + + String response = dl.download(apiUrl); + JSONObject responseObject = new JSONObject(response); + + nextUrl = responseObject.getString("next_href") + + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&linked_partitioning=1"; + + JSONArray responseCollection = responseObject.getJSONArray("collection"); + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject track = responseCollection.getJSONObject(i); + collector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } + return collector; + } + + @Override + public long getSubscriberCount() { + return channel.getLong("followers_count"); + } + + @Override + public String getFeedUrl() throws ParsingException { + return null; + } + + @Override + public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { + if (nextUrl.equals("")) { + throw new ExtractionException("Channel doesn't have more streams"); + } + + StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(nextUrl); + JSONObject responseObject = new JSONObject(response); + + nextUrl = responseObject.getString("next_href") + + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&linked_partitioning=1"; + + JSONArray responseCollection = responseObject.getJSONArray("collection"); + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject track = responseCollection.getJSONObject(i); + collector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } + return collector; + } +} diff --git a/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/services/soundcloud/SoundcloudChannelInfoItemExtractor.java new file mode 100644 index 000000000..160de129a --- /dev/null +++ b/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.json.JSONObject; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; + +public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { + private JSONObject searchResult; + + public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) { + this.searchResult = searchResult; + } + + @Override + public String getThumbnailUrl() { + return searchResult.getString("avatar_url"); + } + + @Override + public String getChannelName() { + return searchResult.getString("username"); + } + + @Override + public String getWebPageUrl() { + return searchResult.getString("permalink_url"); + } + + @Override + public long getSubscriberCount() { + return searchResult.getLong("followers_count"); + } + + @Override + public long getViewCount() { + return searchResult.getLong("track_count"); + } + + @Override + public String getDescription() { + return searchResult.getString("description"); + } +} diff --git a/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/services/soundcloud/SoundcloudChannelUrlIdHandler.java new file mode 100644 index 000000000..be6c20754 --- /dev/null +++ b/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { + + private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler(); + + public static SoundcloudChannelUrlIdHandler getInstance() { + return instance; + } + + @Override + public String getUrl(String channelId) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId + + "?client_id=" + SoundcloudParsingHelper.clientId()); + JSONObject responseObject = new JSONObject(response); + + return responseObject.getString("permalink_url"); + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String getId(String siteUrl) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(siteUrl); + Document doc = Jsoup.parse(response); + + Element androidElement = doc.select("meta[property=al:android:url]").first(); + String id = androidElement.attr("content").substring(19); + + return id; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String cleanUrl(String siteUrl) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(siteUrl); + Document doc = Jsoup.parse(response); + + Element ogElement = doc.select("meta[property=og:url]").first(); + String url = ogElement.attr("content"); + + return url; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public boolean acceptUrl(String channelUrl) { + String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; + return Parser.isMatch(regex, channelUrl.toLowerCase()); + + } +} diff --git a/services/soundcloud/SoundcloudParsingHelper.java b/services/soundcloud/SoundcloudParsingHelper.java new file mode 100644 index 000000000..45af5c29e --- /dev/null +++ b/services/soundcloud/SoundcloudParsingHelper.java @@ -0,0 +1,80 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +public class SoundcloudParsingHelper { + private SoundcloudParsingHelper() { + } + + public static final String clientId() throws ReCaptchaException, IOException, RegexException { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download("https://soundcloud.com"); + Document doc = Jsoup.parse(response); + + Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first(); + String js = dl.download(jsElement.attr("src")); + + String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js); + return clientId; + } + + public static String toTimeAgoString(String time) throws ParsingException { + try { + List times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30), + TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1), + TimeUnit.SECONDS.toMillis(1)); + List timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second"); + + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime(); + + StringBuilder timeAgoString = new StringBuilder(); + + for (int i = 0; i < times.size(); i++) { + Long current = times.get(i); + long currentAmount = timeAgo / current; + if (currentAmount > 0) { + timeAgoString.append(currentAmount).append(" ").append(timesString.get(i)) + .append(currentAmount != 1 ? "s ago" : " ago"); + break; + } + } + if (timeAgoString.toString().equals("")) { + timeAgoString.append("Just now"); + } + return timeAgoString.toString(); + } catch (ParseException e) { + throw new ParsingException(e.getMessage(), e); + } + } + + public static String toDateString(String time) throws ParsingException { + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + Date date = dateFormat.parse(time); + SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return newDateFormat.format(date); + } catch (ParseException e) { + throw new ParsingException(e.getMessage(), e); + } + } + +} diff --git a/services/soundcloud/SoundcloudPlaylistExtractor.java b/services/soundcloud/SoundcloudPlaylistExtractor.java new file mode 100644 index 000000000..01a0bee4e --- /dev/null +++ b/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -0,0 +1,129 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +@SuppressWarnings("WeakerAccess") +public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + private String playlistId; + private JSONObject playlist; + private List nextTracks; + + public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException { + super(urlIdHandler, url, serviceId); + + Downloader dl = NewPipe.getDownloader(); + playlistId = urlIdHandler.getId(url); + + String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + playlist = new JSONObject(response); + } + + @Override + public String getPlaylistId() { + return playlistId; + } + + @Override + public String getPlaylistName() { + return playlist.getString("title"); + } + + @Override + public String getAvatarUrl() { + return playlist.getString("artwork_url"); + } + + @Override + public String getBannerUrl() { + return null; + } + + @Override + public String getUploaderUrl() { + return playlist.getJSONObject("user").getString("permalink_url"); + } + + @Override + public String getUploaderName() { + return playlist.getJSONObject("user").getString("username"); + } + + @Override + public String getUploaderAvatarUrl() { + return playlist.getJSONObject("user").getString("avatar_url"); + } + + @Override + public long getStreamsCount() { + return playlist.getLong("track_count"); + } + + @Override + public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException { + StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + Downloader dl = NewPipe.getDownloader(); + + String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + JSONObject responseObject = new JSONObject(response); + JSONArray responseCollection = responseObject.getJSONArray("collection"); + + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject track = responseCollection.getJSONObject(i); + try { + collector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } catch (Exception e) { + nextTracks.add(track.getString("id")); + } + } + return collector; + } + + @Override + public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException { + if (nextTracks.equals(null)) { + return null; + } + + StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + Downloader dl = NewPipe.getDownloader(); + + // TODO: Do this per 10 tracks, instead of all tracks at once + String apiUrl = "https://api-v2.soundcloud.com/tracks?ids="; + for (String id : nextTracks) { + apiUrl += id; + if (!id.equals(nextTracks.get(nextTracks.size() - 1))) { + apiUrl += ","; + } + } + apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + JSONObject responseObject = new JSONObject(response); + JSONArray responseCollection = responseObject.getJSONArray("collection"); + + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject track = responseCollection.getJSONObject(i); + collector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } + nextTracks = null; + return collector; + } +} diff --git a/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java new file mode 100644 index 000000000..8244a8146 --- /dev/null +++ b/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -0,0 +1,75 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { + + private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler(); + + public static SoundcloudPlaylistUrlIdHandler getInstance() { + return instance; + } + + @Override + public String getUrl(String listId) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId + + "?client_id=" + SoundcloudParsingHelper.clientId()); + JSONObject responseObject = new JSONObject(response); + + return responseObject.getString("permalink_url"); + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String getId(String url) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(url); + Document doc = Jsoup.parse(response); + + Element androidElement = doc.select("meta[property=al:android:url]").first(); + String id = androidElement.attr("content").substring(23); + + return id; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String cleanUrl(String complexUrl) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(complexUrl); + Document doc = Jsoup.parse(response); + + Element ogElement = doc.select("meta[property=og:url]").first(); + String url = ogElement.attr("content"); + + return url; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public boolean acceptUrl(String videoUrl) { + String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+/sets/[0-9a-z_-]+/?([#?].*)?$"; + return Parser.isMatch(regex, videoUrl.toLowerCase()); + } +} diff --git a/services/soundcloud/SoundcloudSearchEngine.java b/services/soundcloud/SoundcloudSearchEngine.java new file mode 100644 index 000000000..8837b933b --- /dev/null +++ b/services/soundcloud/SoundcloudSearchEngine.java @@ -0,0 +1,61 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.EnumSet; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; +import org.schabi.newpipe.extractor.search.SearchEngine; + +public class SoundcloudSearchEngine extends SearchEngine { + public static final String CHARSET_UTF_8 = "UTF-8"; + + public SoundcloudSearchEngine(int serviceId) { + super(serviceId); + } + + @Override + public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet filter) throws IOException, ExtractionException { + InfoItemSearchCollector collector = getInfoItemSearchCollector(); + + Downloader downloader = NewPipe.getDownloader(); + + String url = "https://api-v2.soundcloud.com/search"; + + if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { + url += "/tracks"; + } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { + url += "/users"; + } + + url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=10" + + "&offset=" + Integer.toString(page * 10); + + String searchJson = downloader.download(url); + JSONObject search = new JSONObject(searchJson); + JSONArray searchCollection = search.getJSONArray("collection"); + + if (searchCollection.length() == 0) { + throw new NothingFoundException("Nothing found"); + } + + for (int i = 0; i < searchCollection.length(); i++) { + JSONObject searchResult = searchCollection.getJSONObject(i); + String kind = searchResult.getString("kind"); + if (kind.equals("user")) { + collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); + } else if (kind.equals("track")) { + collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); + } + } + + return collector; + } +} diff --git a/services/soundcloud/SoundcloudService.java b/services/soundcloud/SoundcloudService.java new file mode 100644 index 000000000..9a0ba4311 --- /dev/null +++ b/services/soundcloud/SoundcloudService.java @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.stream.StreamExtractor; + +import java.io.IOException; + +public class SoundcloudService extends StreamingService { + + public SoundcloudService(int id) { + super(id); + } + + @Override + public ServiceInfo getServiceInfo() { + ServiceInfo serviceInfo = new ServiceInfo(); + serviceInfo.name = "Soundcloud"; + return serviceInfo; + } + + @Override + public StreamExtractor getStreamExtractorInstance(String url) + throws ExtractionException, IOException { + UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance(); + if (urlIdHandler.acceptUrl(url)) { + return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId()); + } else { + throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL"); + } + } + + @Override + public SearchEngine getSearchEngineInstance() { + return new SoundcloudSearchEngine(getServiceId()); + } + + @Override + public UrlIdHandler getStreamUrlIdHandlerInstance() { + return SoundcloudStreamUrlIdHandler.getInstance(); + } + + @Override + public UrlIdHandler getChannelUrlIdHandlerInstance() { + return SoundcloudChannelUrlIdHandler.getInstance(); + } + + + @Override + public UrlIdHandler getPlaylistUrlIdHandlerInstance() { + return SoundcloudPlaylistUrlIdHandler.getInstance(); + } + + @Override + public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException { + return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId()); + } + + @Override + public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException { + return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId()); + } + + @Override + public SuggestionExtractor getSuggestionExtractorInstance() { + return new SoundcloudSuggestionExtractor(getServiceId()); + } +} diff --git a/services/soundcloud/SoundcloudStreamExtractor.java b/services/soundcloud/SoundcloudStreamExtractor.java new file mode 100644 index 000000000..61eaa5c53 --- /dev/null +++ b/services/soundcloud/SoundcloudStreamExtractor.java @@ -0,0 +1,230 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.util.List; +import java.util.Vector; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +public class SoundcloudStreamExtractor extends StreamExtractor { + private String pageUrl; + private String trackId; + private JSONObject track; + + public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException { + super(urlIdHandler, pageUrl, serviceId); + + Downloader dl = NewPipe.getDownloader(); + + trackId = urlIdHandler.getId(pageUrl); + String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + track = new JSONObject(response); + + if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) { + throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy")); + } + } + + @Override + public String getId() { + return trackId; + } + + @Override + public String getTitle() { + return track.getString("title"); + } + + @Override + public String getDescription() { + return track.getString("description"); + } + + @Override + public String getUploader() { + return track.getJSONObject("user").getString("username"); + } + + @Override + public int getLength() { + return track.getInt("duration") / 1000; + } + + @Override + public long getViewCount() { + return track.getLong("playback_count"); + } + + @Override + public String getUploadDate() throws ParsingException { + return SoundcloudParsingHelper.toDateString(track.getString("created_at")); + } + + @Override + public String getThumbnailUrl() { + return track.getString("artwork_url"); + } + + @Override + public String getUploaderThumbnailUrl() { + return track.getJSONObject("user").getString("avatar_url"); + } + + @Override + public String getDashMpdUrl() { + return null; + } + + @Override + public List getAudioStreams() throws ReCaptchaException, IOException, RegexException { + Vector audioStreams = new Vector<>(); + Downloader dl = NewPipe.getDownloader(); + + String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams" + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + JSONObject responseObject = new JSONObject(response); + + AudioStream audioStream = new AudioStream(responseObject.getString("http_mp3_128_url"), MediaFormat.MP3.id, 128); + audioStreams.add(audioStream); + + return audioStreams; + } + + @Override + public List getVideoStreams() { + return null; + } + + @Override + public List getVideoOnlyStreams() { + return null; + } + + @Override + public int getTimeStamp() throws ParsingException { + String timeStamp; + try { + timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + } catch (Parser.RegexException e) { + // catch this instantly since an url does not necessarily have to have a time stamp + + // -2 because well the testing system will then know its the regex that failed :/ + // not good i know + return -2; + } + + if (!timeStamp.isEmpty()) { + try { + String secondsString = ""; + String minutesString = ""; + String hoursString = ""; + try { + secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp); + minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp); + hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp); + } catch (Exception e) { + //it could be that time is given in another method + if (secondsString.isEmpty() //if nothing was got, + && minutesString.isEmpty()//treat as unlabelled seconds + && hoursString.isEmpty()) { + secondsString = Parser.matchGroup1("t=(\\d+)", timeStamp); + } + } + + int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString); + int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString); + int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString); + + //don't trust BODMAS! + return seconds + (60 * minutes) + (3600 * hours); + //Log.d(TAG, "derived timestamp value:"+ret); + //the ordering varies internationally + } catch (ParsingException e) { + throw new ParsingException("Could not get timestamp.", e); + } + } else { + return 0; + } + } + + @Override + public int getAgeLimit() { + return 0; + } + + @Override + public String getAverageRating() { + return null; + } + + @Override + public int getLikeCount() { + return track.getInt("likes_count"); + } + + @Override + public int getDislikeCount() { + return 0; + } + + @Override + public StreamInfoItemExtractor getNextVideo() { + return null; + } + + @Override + public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException { + StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + Downloader dl = NewPipe.getDownloader(); + + String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related" + + "?client_id=" + SoundcloudParsingHelper.clientId(); + + String response = dl.download(apiUrl); + JSONObject responseObject = new JSONObject(response); + JSONArray responseCollection = responseObject.getJSONArray("collection"); + + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject relatedVideo = responseCollection.getJSONObject(i); + collector.commit(new SoundcloudStreamInfoItemExtractor(relatedVideo)); + } + return collector; + } + + @Override + public String getChannelUrl() { + return track.getJSONObject("user").getString("permalink_url"); + } + + @Override + public StreamType getStreamType() { + return StreamType.AUDIO_STREAM; + } + + @Override + public String getErrorMessage() { + return null; + } +} diff --git a/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/services/soundcloud/SoundcloudStreamInfoItemExtractor.java new file mode 100644 index 000000000..001315c09 --- /dev/null +++ b/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -0,0 +1,60 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.json.JSONObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { + + private final JSONObject searchResult; + + public SoundcloudStreamInfoItemExtractor(JSONObject searchResult) { + this.searchResult = searchResult; + } + + @Override + public String getWebPageUrl() { + return searchResult.getString("permalink_url"); + } + + @Override + public String getTitle() { + return searchResult.getString("title"); + } + + @Override + public int getDuration() { + return searchResult.getInt("duration") / 1000; + } + + @Override + public String getUploader() { + return searchResult.getJSONObject("user").getString("username"); + } + + @Override + public String getUploadDate() throws ParsingException { + return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at")); + } + + @Override + public long getViewCount() { + return searchResult.getLong("playback_count"); + } + + @Override + public String getThumbnailUrl() { + return searchResult.getString("artwork_url"); + } + + @Override + public StreamType getStreamType() { + return StreamType.AUDIO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } +} diff --git a/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/services/soundcloud/SoundcloudStreamUrlIdHandler.java new file mode 100644 index 000000000..2badb8ee5 --- /dev/null +++ b/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { + + private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); + private SoundcloudStreamUrlIdHandler() { + } + + public static SoundcloudStreamUrlIdHandler getInstance() { + return instance; + } + + @Override + public String getUrl(String videoId) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId + + "?client_id=" + SoundcloudParsingHelper.clientId()); + JSONObject responseObject = new JSONObject(response); + + return responseObject.getString("permalink_url"); + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String getId(String url) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(url); + Document doc = Jsoup.parse(response); + + Element androidElement = doc.select("meta[property=al:android:url]").first(); + String id = androidElement.attr("content").substring(20); + + return id; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public String cleanUrl(String complexUrl) throws ParsingException { + try { + Downloader dl = NewPipe.getDownloader(); + + String response = dl.download(complexUrl); + Document doc = Jsoup.parse(response); + + Element ogElement = doc.select("meta[property=og:url]").first(); + String url = ogElement.attr("content"); + + return url; + } catch (Exception e) { + throw new ParsingException(e.getMessage(), e); + } + } + + @Override + public boolean acceptUrl(String videoUrl) { + String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; + return Parser.isMatch(regex, videoUrl.toLowerCase()); + } +} diff --git a/services/soundcloud/SoundcloudSuggestionExtractor.java b/services/soundcloud/SoundcloudSuggestionExtractor.java new file mode 100644 index 000000000..6af0fdbdd --- /dev/null +++ b/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +public class SoundcloudSuggestionExtractor extends SuggestionExtractor { + + public static final String CHARSET_UTF_8 = "UTF-8"; + + public SoundcloudSuggestionExtractor(int serviceId) { + super(serviceId); + } + + @Override + public List suggestionList(String query, String contentCountry) throws RegexException, ReCaptchaException, IOException { + List suggestions = new ArrayList<>(); + + Downloader dl = NewPipe.getDownloader(); + + String url = "https://api-v2.soundcloud.com/search/queries" + + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=10"; + + String response = dl.download(url); + JSONObject responseObject = new JSONObject(response); + JSONArray responseCollection = responseObject.getJSONArray("collection"); + + for (int i = 0; i < responseCollection.length(); i++) { + JSONObject suggestion = responseCollection.getJSONObject(i); + suggestions.add(suggestion.getString("query")); + } + + return suggestions; + } +} diff --git a/stream/StreamExtractor.java b/stream/StreamExtractor.java index 308816bf2..1cf4c1fe5 100644 --- a/stream/StreamExtractor.java +++ b/stream/StreamExtractor.java @@ -23,7 +23,9 @@ package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import java.io.IOException; import java.util.List; /** @@ -46,7 +48,7 @@ public abstract class StreamExtractor extends Extractor { public abstract String getUploadDate() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException; public abstract String getUploaderThumbnailUrl() throws ParsingException; - public abstract List getAudioStreams() throws ParsingException; + public abstract List getAudioStreams() throws ParsingException, ReCaptchaException, IOException; public abstract List getVideoStreams() throws ParsingException; public abstract List getVideoOnlyStreams() throws ParsingException; public abstract String getDashMpdUrl() throws ParsingException; @@ -55,7 +57,7 @@ public abstract class StreamExtractor extends Extractor { public abstract int getLikeCount() throws ParsingException; public abstract int getDislikeCount() throws ParsingException; public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException; - public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException; + public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException; public abstract StreamType getStreamType() throws ParsingException; /** diff --git a/utils/Parser.java b/utils/Parser.java index b59bd9504..2db762462 100644 --- a/utils/Parser.java +++ b/utils/Parser.java @@ -63,6 +63,15 @@ public class Parser { } } + public static boolean isMatch(String pattern, String input) { + try { + matchGroup1(pattern, input); + return true; + } catch (RegexException e) { + return false; + } + } + public static Map compatParseMap(final String input) throws UnsupportedEncodingException { Map map = new HashMap<>(); for (String arg : input.split("&")) {