be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1081 from TeamNewPipe/fix/sc/search-next-page 

[SoundCloud] Detect whether there are any more search results
This commit is contained in:
Stypox 2023-08-06 11:49:35 +02:00 committed by GitHub
parent 7c70fef197 2947257111
commit 8fb6ba36fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 82 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -1,14 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -28,6 +23,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
@ -38,7 +34,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public final class SoundcloudParsingHelper {
private static String clientId;
@ -200,6 +198,7 @@ public final class SoundcloudParsingHelper {
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
@ -221,17 +220,7 @@ public final class SoundcloudParsingHelper {
}
}
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
} catch (final Exception ignored) {
nextPageUrl = "";
}
return nextPageUrl;
return getNextPageUrl(responseObject);
}
/**
@ -261,6 +250,7 @@ public final class SoundcloudParsingHelper {
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
final boolean charts) throws IOException,
@ -288,17 +278,20 @@ public final class SoundcloudParsingHelper {
}
}
String nextPageUrl;
return getNextPageUrl(responseObject);
}
@Nonnull
private static String getNextPageUrl(@Nonnull final JsonObject response) {
try {
nextPageUrl = responseObject.getString("next_href");
String nextPageUrl = response.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
return nextPageUrl;
} catch (final Exception ignored) {
nextPageUrl = "";
return "";
}
return nextPageUrl;
}
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import java.io.IOException;
@ -33,22 +34,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return getPage(getUrl());
}
@Override
@ -57,9 +43,14 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
return getPage(page.getUrl());
}
@Nonnull
private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
throws ParsingException, IOException, ReCaptchaException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(page.getUrl());
final Response response = downloader.get(url);
final JsonObject json;
try {
@ -72,8 +63,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
}
@Override

47
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -33,7 +33,9 @@ import java.util.function.IntUnaryOperator;
import javax.annotation.Nonnull;
public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray initialSearchCollection;
private JsonObject initialSearchObject;
private static final String COLLECTION = "collection";
private static final String TOTAL_RESULTS = "total_results";
public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
@ -60,9 +62,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(
collectItems(initialSearchCollection),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
} else {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)), null);
}
}
@Override
@ -74,17 +82,23 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final Downloader dl = getDownloader();
final JsonArray searchCollection;
final int totalResults;
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
final JsonObject result = JsonParser.object().from(response);
searchCollection = result.getArray(COLLECTION);
totalResults = result.getInt(TOTAL_RESULTS);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
if (getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE < totalResults) {
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
}
return new InfoItemsPage<>(collectItems(searchCollection), null);
}
@Override
@ -94,12 +108,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
initialSearchCollection = JsonParser.object().from(response).getArray("collection");
initialSearchObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
if (initialSearchCollection.isEmpty()) {
if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
throw new SearchExtractor.NothingFoundException("Nothing found");
}
}
@ -133,13 +147,20 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private Page getNextPageFromCurrentUrl(final String currentUrl,
final IntUnaryOperator newPageOffsetCalculator)
throws MalformedURLException, UnsupportedEncodingException {
final int currentPageOffset = Integer.parseInt(
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
throws ParsingException {
final int currentPageOffset = getOffsetFromUrl(currentUrl);
return new Page(
currentUrl.replace(
"&offset=" + currentPageOffset,
"&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
}
private int getOffsetFromUrl(final String url) throws ParsingException {
try {
return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
} catch (MalformedURLException | UnsupportedEncodingException e) {
throw new ParsingException("Could not get offset from page URL", e);
}
}
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
@ -181,4 +182,27 @@ public class SoundcloudSearchExtractorTest {
assertTrue(verified);
}
}
public static class NoNextPage extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "Dan at hor#berlgbd";
@BeforeAll
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getSearchExtractor(QUERY);
extractor.fetchPage();
}
@Override public boolean expectedHasMoreItems() { return false; }
@Override public SearchExtractor extractor() throws Exception { return extractor; }
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return QUERY; }
@Override public String expectedId() throws Exception { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
}