be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[SoundCloud] Detect whether there are any more search results 

Add test for this edge case.
This commit is contained in:
TobiGr 2023-07-14 17:43:09 +02:00
parent 5492343b8e
commit 2fb9922a15
2 changed files with 60 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -33,7 +33,9 @@ import java.util.function.IntUnaryOperator;
import javax.annotation.Nonnull;
public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray initialSearchCollection;
private JsonObject initialSearchObject;
private static final String COLLECTION = "collection";
private static final String TOTAL_RESULTS = "total_results";
public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
@ -60,9 +62,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(
collectItems(initialSearchCollection),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
} else {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)), null);
}
}
@Override
@ -74,17 +82,29 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final Downloader dl = getDownloader();
final JsonArray searchCollection;
final int totalResults;
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
final JsonObject result = JsonParser.object().from(response);
searchCollection = result.getArray(COLLECTION);
totalResults = result.getInt(TOTAL_RESULTS);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
final boolean hasNextPage;
try {
hasNextPage = getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE <= totalResults;
} catch (MalformedURLException | UnsupportedEncodingException e) {
throw new ParsingException("Could not get offset from page URL", e);
}
if (hasNextPage) {
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
}
return new InfoItemsPage<>(collectItems(searchCollection), null);
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
}
@Override
@ -94,12 +114,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
initialSearchCollection = JsonParser.object().from(response).getArray("collection");
initialSearchObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
if (initialSearchCollection.isEmpty()) {
if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
throw new SearchExtractor.NothingFoundException("Nothing found");
}
}
@ -134,12 +154,16 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private Page getNextPageFromCurrentUrl(final String currentUrl,
final IntUnaryOperator newPageOffsetCalculator)
throws MalformedURLException, UnsupportedEncodingException {
final int currentPageOffset = Integer.parseInt(
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
final int currentPageOffset = getOffsetFromUrl(currentUrl);
return new Page(
currentUrl.replace(
"&offset=" + currentPageOffset,
"&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
}
private int getOffsetFromUrl(final String url)
throws MalformedURLException, UnsupportedEncodingException {
return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
}
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
@ -181,4 +182,27 @@ public class SoundcloudSearchExtractorTest {
assertTrue(verified);
}
}
public static class NoNextPage extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "Dan at hor#berlgbd";
@BeforeAll
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getSearchExtractor(QUERY);
extractor.fetchPage();
}
@Override public boolean expectedHasMoreItems() { return false; }
@Override public SearchExtractor extractor() throws Exception { return extractor; }
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return QUERY; }
@Override public String expectedId() throws Exception { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
}