diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java index 7d649732b..d3bc435cc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java @@ -15,6 +15,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Random; import javax.annotation.Nonnull; @@ -37,17 +38,31 @@ import javax.annotation.Nonnull; */ class RecordingDownloader extends Downloader { - public final static String FILE_NAME_PREFIX = "generated_mock_"; + public static final String FILE_NAME_PREFIX = "generated_mock_"; // From https://stackoverflow.com/a/15875500/13516981 - private final static String IP_V4_PATTERN = + private static final String IP_V4_PATTERN = "(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"; private int index = 0; private final String path; + // try to prevent ReCaptchaExceptions / rate limits by tracking and throttling the requests /** - * Creates the folder described by {@code stringPath} if it does not exists. + * The maximum number of requests per 20 seconds which are executed + * by the {@link RecordingDownloader}. + * 20 seconds is used as upper bound because the rate limit can be triggered within 30 seconds + * and hitting the rate limit should be prevented because it comes with a bigger delay. + * The values can be adjusted when executing the downloader and running into problems. + *
TODO: Allow adjusting the value by setting a param in the gradle command
+ */ + private static final int MAX_REQUESTS_PER_20_SECONDS = 30; + private static final long[] requestTimes = new long[MAX_REQUESTS_PER_20_SECONDS]; + private static int requestTimesCursor = -1; + private static final Random throttleRandom = new Random(); + + /** + * Creates the folder described by {@code stringPath} if it does not exist. * Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}. * @param stringPath Path to the folder where the json files will be saved to. */ @@ -69,6 +84,48 @@ class RecordingDownloader extends Downloader { @Override public Response execute(@Nonnull final Request request) throws IOException, ReCaptchaException { + + // Delay the execution if the max number of requests per minute is reached + final long currentTime = System.currentTimeMillis(); + // the cursor points to the latest request time and the next position is the oldest one + final int oldestRequestTimeCursor = (requestTimesCursor + 1) % requestTimes.length; + final long oldestRequestTime = requestTimes[oldestRequestTimeCursor]; + if (oldestRequestTime + 20_000 >= currentTime) { + try { + // sleep at least until the oldest request is 20s old, but not more than 20s + final int minSleepTime = (int) (currentTime - oldestRequestTime); + Thread.sleep(minSleepTime + throttleRandom.nextInt(20_000 - minSleepTime)); + } catch (InterruptedException e) { + // handle the exception gracefully because it's not critical for the test + System.err.println("Error while throttling the RecordingDownloader."); + e.printStackTrace(); + } + } + requestTimesCursor = oldestRequestTimeCursor; // the oldest value needs to be overridden + requestTimes[requestTimesCursor] = System.currentTimeMillis(); + + // Handle ReCaptchaExceptions by retrying the request once after a while + try { + return executeRequest(request); + } catch (ReCaptchaException e) { + try { + // sleep for 35-60 seconds to circumvent the rate limit + System.out.println("Throttling the RecordingDownloader to handle a ReCaptcha." + + " Sleeping for 35-60 seconds."); + Thread.sleep(35_000 + throttleRandom.nextInt(25_000)); + } catch (InterruptedException ie) { + // handle the exception gracefully because it's not critical for the test + System.err.println("Error while throttling the RecordingDownloader."); + ie.printStackTrace(); + e.printStackTrace(); + } + return executeRequest(request); + } + } + + @Nonnull + private Response executeRequest(@Nonnull final Request request) throws IOException, + ReCaptchaException { final Downloader downloader = DownloaderTestImpl.getInstance(); Response response = downloader.execute(request); String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");