lasers/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor synced 2024-11-26 03:59:16 +01:00
Code Issues Projects Releases Wiki Activity

Throw upon trying to fetch unencrypted URL

Browse Source
This commit is contained in:
Fynn Godau 2024-05-24 19:24:41 +02:00
parent fafd471606
commit bff7518cfe
4 changed files with 45 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -25,10 +26,10 @@ public abstract class Downloader {
* localization. It should only be used when the resource that will be fetched won't be affected
* by the localization.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the GET request
*/
public Response get(final String url) throws IOException, ReCaptchaException {
public final Response get(final String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}
@ -37,11 +38,11 @@ public abstract class Downloader {
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url, final Localization localization)
public final Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException {
return get(url, null, localization);
}
@ -49,12 +50,12 @@ public abstract class Downloader {
/**
* Do a GET request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request
*/
public Response get(final String url, @Nullable final Map<String, List<String>> headers)
public final Response get(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
@ -64,17 +65,17 @@ public abstract class Downloader {
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url,
public final Response get(final String url,
@Nullable final Map<String, List<String>> headers,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.get(url)
.headers(headers)
.localization(localization)
@ -84,24 +85,24 @@ public abstract class Downloader {
/**
* Do a HEAD request.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the HEAD request
*/
public Response head(final String url) throws IOException, ReCaptchaException {
public final Response head(final String url) throws IOException, ReCaptchaException {
return head(url, null);
}
/**
* Do a HEAD request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request
*/
public Response head(final String url, @Nullable final Map<String, List<String>> headers)
public final Response head(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.head(url)
.headers(headers)
.build());
@ -110,13 +111,13 @@ public abstract class Downloader {
/**
* Do a POST request with the specified headers, sending the data array.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
@ -128,19 +129,19 @@ public abstract class Downloader {
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
@ -151,7 +152,7 @@ public abstract class Downloader {
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@ -161,7 +162,7 @@ public abstract class Downloader {
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization,
@ -179,7 +180,7 @@ public abstract class Downloader {
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@ -188,7 +189,7 @@ public abstract class Downloader {
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final String contentType)
@ -201,7 +202,7 @@ public abstract class Downloader {
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@ -209,7 +210,7 @@ public abstract class Downloader {
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
@ -221,14 +222,14 @@ public abstract class Downloader {
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
@ -236,11 +237,24 @@ public abstract class Downloader {
NewPipe.getPreferredLocalization());
}
public final Response executeIfHttps(final @Nonnull Request request)
throws IOException, ReCaptchaException {
if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
throw new IOException(
"All queries must be made using HTTPS. Extractors must guarantee "
+ "that HTTPS links are provided."
);
} else {
return execute(request);
}
}
/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
public abstract Response execute(@Nonnull Request request)
protected abstract Response execute(@Nonnull Request request)
throws IOException, ReCaptchaException;
}

2
extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
View File

@ -47,7 +47,7 @@ public final class DownloaderTestImpl extends Downloader {
}
@Override
public Response execute(@Nonnull final Request request)
protected Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();

2
extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
View File

@ -46,7 +46,7 @@ class MockDownloader extends Downloader {
}
@Override
public Response execute(@Nonnull final Request request) {
protected Response execute(@Nonnull final Request request) {
final Response result = mocks.get(request);
if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request

2
extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
View File

@ -70,7 +70,7 @@ class RecordingDownloader extends Downloader {
public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request);
Response response = downloader.executeIfHttps(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response(
response.responseCode(),