be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Throw upon trying to fetch unencrypted URL

This commit is contained in:
Fynn Godau 2024-05-24 19:24:41 +02:00
parent fafd471606
commit bff7518cfe
4 changed files with 45 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -25,10 +26,10 @@ public abstract class Downloader {
* localization. It should only be used when the resource that will be fetched won't be affected * localization. It should only be used when the resource that will be fetched won't be affected
* by the localization. * by the localization.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the GET request * @return the result of the GET request
*/ */
public Response get(final String url) throws IOException, ReCaptchaException { public final Response get(final String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization()); return get(url, null, NewPipe.getPreferredLocalization());
} }
@ -37,11 +38,11 @@ public abstract class Downloader {
* <br> * <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter. * It will set the {@code Accept-Language} header to the language of the localization parameter.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param localization the source of the value of the {@code Accept-Language} header * @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request * @return the result of the GET request
*/ */
public Response get(final String url, final Localization localization) public final Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
return get(url, null, localization); return get(url, null, localization);
} }
@ -49,12 +50,12 @@ public abstract class Downloader {
/** /**
* Do a GET request with the specified headers. * Do a GET request with the specified headers.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request * @return the result of the GET request
*/ */
public Response get(final String url, @Nullable final Map<String, List<String>> headers) public final Response get(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization()); return get(url, headers, NewPipe.getPreferredLocalization());
} }
@ -64,17 +65,17 @@ public abstract class Downloader {
* <br> * <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter. * It will set the {@code Accept-Language} header to the language of the localization parameter.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header * @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request * @return the result of the GET request
*/ */
public Response get(final String url, public final Response get(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
final Localization localization) final Localization localization)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
return execute(Request.newBuilder() return executeIfHttps(Request.newBuilder()
.get(url) .get(url)
.headers(headers) .headers(headers)
.localization(localization) .localization(localization)
@ -84,24 +85,24 @@ public abstract class Downloader {
/** /**
* Do a HEAD request. * Do a HEAD request.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the HEAD request * @return the result of the HEAD request
*/ */
public Response head(final String url) throws IOException, ReCaptchaException { public final Response head(final String url) throws IOException, ReCaptchaException {
return head(url, null); return head(url, null);
} }
/** /**
* Do a HEAD request with the specified headers. * Do a HEAD request with the specified headers.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request * @return the result of the HEAD request
*/ */
public Response head(final String url, @Nullable final Map<String, List<String>> headers) public final Response head(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
return execute(Request.newBuilder() return executeIfHttps(Request.newBuilder()
.head(url) .head(url)
.headers(headers) .headers(headers)
.build()); .build());
@ -110,13 +111,13 @@ public abstract class Downloader {
/** /**
* Do a POST request with the specified headers, sending the data array. * Do a POST request with the specified headers, sending the data array.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request * @return the result of the POST request
*/ */
public Response post(final String url, public final Response post(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend) @Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
@ -128,19 +129,19 @@ public abstract class Downloader {
* <br> * <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter. * It will set the {@code Accept-Language} header to the language of the localization parameter.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header * @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request * @return the result of the POST request
*/ */
public Response post(final String url, public final Response post(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend, @Nullable final byte[] dataToSend,
final Localization localization) final Localization localization)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
return execute(Request.newBuilder() return executeIfHttps(Request.newBuilder()
.post(url, dataToSend) .post(url, dataToSend)
.headers(headers) .headers(headers)
.localization(localization) .localization(localization)
@ -151,7 +152,7 @@ public abstract class Downloader {
* Convenient method to send a POST request using the specified value of the * Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}. * {@code Content-Type} header with a given {@link Localization}.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
@ -161,7 +162,7 @@ public abstract class Downloader {
* @return the result of the POST request * @return the result of the POST request
* @see #post(String, Map, byte[], Localization) * @see #post(String, Map, byte[], Localization)
*/ */
public Response postWithContentType(final String url, public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend, @Nullable final byte[] dataToSend,
final Localization localization, final Localization localization,
@ -179,7 +180,7 @@ public abstract class Downloader {
* Convenient method to send a POST request using the specified value of the * Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header. * {@code Content-Type} header.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
@ -188,7 +189,7 @@ public abstract class Downloader {
* @return the result of the POST request * @return the result of the POST request
* @see #post(String, Map, byte[], Localization) * @see #post(String, Map, byte[], Localization)
*/ */
public Response postWithContentType(final String url, public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend, @Nullable final byte[] dataToSend,
final String contentType) final String contentType)
@ -201,7 +202,7 @@ public abstract class Downloader {
* Convenient method to send a POST request the JSON mime type as the value of the * Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}. * {@code Content-Type} header with a given {@link Localization}.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
@ -209,7 +210,7 @@ public abstract class Downloader {
* @return the result of the POST request * @return the result of the POST request
* @see #post(String, Map, byte[], Localization) * @see #post(String, Map, byte[], Localization)
*/ */
public Response postWithContentTypeJson(final String url, public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend, @Nullable final byte[] dataToSend,
final Localization localization) final Localization localization)
@ -221,14 +222,14 @@ public abstract class Downloader {
* Convenient method to send a POST request the JSON mime type as the value of the * Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header. * {@code Content-Type} header.
* *
* @param url the URL that is pointing to the wanted resource * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request. * @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these. * Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request. * @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request * @return the result of the POST request
* @see #post(String, Map, byte[], Localization) * @see #post(String, Map, byte[], Localization)
*/ */
public Response postWithContentTypeJson(final String url, public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers, @Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend) @Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
@ -236,11 +237,24 @@ public abstract class Downloader {
NewPipe.getPreferredLocalization()); NewPipe.getPreferredLocalization());
} }
public final Response executeIfHttps(final @Nonnull Request request)
throws IOException, ReCaptchaException {
if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
throw new IOException(
"All queries must be made using HTTPS. Extractors must guarantee "
+ "that HTTPS links are provided."
);
} else {
return execute(request);
}
}
/** /**
* Do a request using the specified {@link Request} object. * Do a request using the specified {@link Request} object.
* *
* @return the result of the request * @return the result of the request
*/ */
public abstract Response execute(@Nonnull Request request) protected abstract Response execute(@Nonnull Request request)
throws IOException, ReCaptchaException; throws IOException, ReCaptchaException;
} }

2
extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
View File

@ -47,7 +47,7 @@ public final class DownloaderTestImpl extends Downloader {
} }
@Override @Override
public Response execute(@Nonnull final Request request) protected Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod(); final String httpMethod = request.httpMethod();
final String url = request.url(); final String url = request.url();

2
extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
View File

@ -46,7 +46,7 @@ class MockDownloader extends Downloader {
} }
@Override @Override
public Response execute(@Nonnull final Request request) { protected Response execute(@Nonnull final Request request) {
final Response result = mocks.get(request); final Response result = mocks.get(request);
if (result == null) { if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request throw new NullPointerException("No mock response for request with url '" + request

2
extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
View File

@ -70,7 +70,7 @@ class RecordingDownloader extends Downloader {
public Response execute(@Nonnull final Request request) throws IOException, public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException { ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance(); final Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request); Response response = downloader.executeIfHttps(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1"); String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response( response = new Response(
response.responseCode(), response.responseCode(),