Add SoundcloudChartsExtractor
This commit is contained in:
wb9688
parent
5119dabb63
commit
e2b7cb9c69
|
|
@ -0,0 +1,63 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
|
||||
public class SoundcloudChartsExtractor extends KioskExtractor {
|
||||
private String url;
|
||||
|
||||
public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
|
||||
super(service, url, nextStreamsUrl);
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fetchPage() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() throws ParsingException {
|
||||
return getUrlIdHandler().getId(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getUrlIdHandler() {
|
||||
return new SoundcloudChartsUrlIdHandler();
|
||||
}
|
||||
|
||||
@Override
|
||||
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
|
||||
if (!hasMoreStreams()) {
|
||||
throw new ExtractionException("Chart doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
|
||||
|
||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
|
||||
String apiUrl = "https://api-v2.soundcloud.com/charts" +
|
||||
"?genre=soundcloud:genres:all-music" +
|
||||
"&client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
if (getType().equals("Top 50")) {
|
||||
apiUrl += "&kind=top";
|
||||
} else {
|
||||
apiUrl += "&kind=new";
|
||||
}
|
||||
|
||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
|
||||
return collector;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
|
||||
public String getUrl(String id) {
|
||||
if (id.equals("Top 50")) {
|
||||
return "https://soundcloud.com/charts/top";
|
||||
} else {
|
||||
return "https://soundcloud.com/charts/new";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(String url) {
|
||||
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
|
||||
return "Top 50";
|
||||
} else {
|
||||
return "New & hot";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String cleanUrl(String url) {
|
||||
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
|
||||
return "https://soundcloud.com/charts/top";
|
||||
} else {
|
||||
return "https://soundcloud.com/charts/new";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean acceptUrl(String url) {
|
||||
return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase());
|
||||
}
|
||||
}
|
||||
|
|
@ -121,13 +121,17 @@ public class SoundcloudParsingHelper {
|
|||
*
|
||||
* @return the next streams url, empty if don't have
|
||||
*/
|
||||
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
|
||||
String response = NewPipe.getDownloader().download(apiUrl);
|
||||
JSONObject responseObject = new JSONObject(response);
|
||||
|
||||
JSONArray responseCollection = responseObject.getJSONArray("collection");
|
||||
for (int i = 0; i < responseCollection.length(); i++) {
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
|
||||
if (charts) {
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track")));
|
||||
} else {
|
||||
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
|
||||
}
|
||||
}
|
||||
|
||||
String nextStreamsUrl;
|
||||
|
|
@ -140,4 +144,8 @@ public class SoundcloudParsingHelper {
|
|||
|
||||
return nextStreamsUrl;
|
||||
}
|
||||
|
||||
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
|
||||
return getStreamsFromApi(collector, apiUrl, false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,17 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.SuggestionExtractor;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskList;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class SoundcloudService extends StreamingService {
|
||||
|
||||
public SoundcloudService(int id, String name) {
|
||||
|
|
@ -61,12 +60,17 @@ public class SoundcloudService extends StreamingService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public KioskList getKioskList() {
|
||||
public KioskList getKioskList() throws ExtractionException {
|
||||
KioskList list = new KioskList(getServiceId());
|
||||
|
||||
// add kiosks here e.g.:
|
||||
//list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler());
|
||||
|
||||
SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler();
|
||||
try {
|
||||
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h);
|
||||
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h);
|
||||
} catch (Exception e) {
|
||||
throw new ExtractionException(e);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
|
||||
/**
|
||||
* Test for {@link SoundcloudChartsUrlIdHandler}
|
||||
*/
|
||||
public class SoundcloudChartsExtractorTest {
|
||||
|
||||
KioskExtractor extractor;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = SoundCloud.getService()
|
||||
.getKioskList()
|
||||
.getExtractorByType("Top 50");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDownloader() throws Exception {
|
||||
assertNotNull(NewPipe.getDownloader());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetName() throws Exception {
|
||||
assertEquals(extractor.getName(), "Top 50");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals(extractor.getId(), "Top 50");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
InfoItemCollector collector = extractor.getStreams();
|
||||
if(!collector.getErrors().isEmpty()) {
|
||||
System.err.println("----------");
|
||||
for(Throwable e : collector.getErrors()) {
|
||||
e.printStackTrace();
|
||||
System.err.println("----------");
|
||||
}
|
||||
}
|
||||
assertTrue("no streams are received",
|
||||
!collector.getItemList().isEmpty()
|
||||
&& collector.getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
assertTrue("has more streams", extractor.hasMoreStreams());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNextStreams() throws Exception {
|
||||
extractor.getStreams();
|
||||
assertFalse("extractor has next streams", extractor.getNextStreams() == null
|
||||
|| extractor.getNextStreams().nextItemsList.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetCleanUrl() throws Exception {
|
||||
assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import static junit.framework.TestCase.assertFalse;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
|
||||
/**
|
||||
* Test for {@link SoundcloudChartsUrlIdHandler}
|
||||
*/
|
||||
public class SoundcloudChartsUrlIdHandlerTest {
|
||||
private SoundcloudChartsUrlIdHandler urlIdHandler;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
urlIdHandler = new SoundcloudChartsUrlIdHandler();
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getUrl() {
|
||||
assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
|
||||
assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getId() {
|
||||
assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
|
||||
assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void acceptUrl() {
|
||||
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
|
||||
assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));
|
||||
assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music"));
|
||||
assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"));
|
||||
|
||||
assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia"));
|
||||
assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf"));
|
||||
assertFalse(urlIdHandler.acceptUrl(" soundcloud.com/charts"));
|
||||
assertFalse(urlIdHandler.acceptUrl(""));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue