mirror of https://github.com/searx/searx
fix proxy requests issues with httpx >= 0.18.x (#2969)
This commit is contained in:
parent
1ea573dc88
commit
79eb2ac69e
|
@ -11,6 +11,6 @@ httpx[http2]==0.19.0
|
||||||
Brotli==1.0.9
|
Brotli==1.0.9
|
||||||
uvloop==0.16.0; python_version >= '3.7'
|
uvloop==0.16.0; python_version >= '3.7'
|
||||||
uvloop==0.14.0; python_version < '3.7'
|
uvloop==0.14.0; python_version < '3.7'
|
||||||
httpx-socks[asyncio]==0.3.1
|
httpx-socks[asyncio]==0.4.1
|
||||||
langdetect==1.0.9
|
langdetect==1.0.9
|
||||||
setproctitle==1.2.2
|
setproctitle==1.2.2
|
||||||
|
|
|
@ -7,7 +7,12 @@ import threading
|
||||||
import httpcore
|
import httpcore
|
||||||
import httpx
|
import httpx
|
||||||
from httpx_socks import AsyncProxyTransport
|
from httpx_socks import AsyncProxyTransport
|
||||||
from python_socks import parse_proxy_url
|
from python_socks import (
|
||||||
|
parse_proxy_url,
|
||||||
|
ProxyConnectionError,
|
||||||
|
ProxyTimeoutError,
|
||||||
|
ProxyError
|
||||||
|
)
|
||||||
import python_socks._errors
|
import python_socks._errors
|
||||||
|
|
||||||
from searx import logger
|
from searx import logger
|
||||||
|
@ -38,7 +43,7 @@ async def close_connections_for_url(connection_pool: httpcore.AsyncConnectionPoo
|
||||||
await connection_pool._remove_from_pool(connection)
|
await connection_pool._remove_from_pool(connection)
|
||||||
try:
|
try:
|
||||||
await connection.aclose()
|
await connection.aclose()
|
||||||
except httpcore.NetworkError as e:
|
except httpx.NetworkError as e:
|
||||||
logger.warning('Error closing an existing connection', exc_info=e)
|
logger.warning('Error closing an existing connection', exc_info=e)
|
||||||
|
|
||||||
|
|
||||||
|
@ -50,50 +55,45 @@ def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http
|
||||||
return SSLCONTEXTS[key]
|
return SSLCONTEXTS[key]
|
||||||
|
|
||||||
|
|
||||||
class AsyncHTTPTransportNoHttp(httpcore.AsyncHTTPTransport):
|
class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
|
||||||
"""Block HTTP request"""
|
"""Block HTTP request"""
|
||||||
|
|
||||||
async def arequest(self, method, url, headers=None, stream=None, ext=None):
|
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None):
|
||||||
raise httpcore.UnsupportedProtocol("HTTP protocol is disabled")
|
raise httpx.UnsupportedProtocol("HTTP protocol is disabled")
|
||||||
|
|
||||||
|
|
||||||
class AsyncProxyTransportFixed(AsyncProxyTransport):
|
class AsyncProxyTransportFixed(AsyncProxyTransport):
|
||||||
"""Fix httpx_socks.AsyncProxyTransport
|
"""Fix httpx_socks.AsyncProxyTransport
|
||||||
|
|
||||||
Map python_socks exceptions to httpcore.ProxyError
|
Map python_socks exceptions to httpx.ProxyError
|
||||||
|
|
||||||
Map socket.gaierror to httpcore.ConnectError
|
Map socket.gaierror to httpx.ConnectError
|
||||||
|
|
||||||
Note: keepalive_expiry is ignored, AsyncProxyTransport should call:
|
Note: keepalive_expiry is ignored, AsyncProxyTransport should call:
|
||||||
* self._keepalive_sweep()
|
* self._keepalive_sweep()
|
||||||
* self._response_closed(self, connection)
|
* self._response_closed(self, connection)
|
||||||
|
|
||||||
Note: AsyncProxyTransport inherit from AsyncConnectionPool
|
Note: AsyncProxyTransport inherit from AsyncConnectionPool
|
||||||
|
|
||||||
Note: the API is going to change on httpx 0.18.0
|
|
||||||
see https://github.com/encode/httpx/pull/1522
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def arequest(self, method, url, headers=None, stream=None, ext=None):
|
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None):
|
||||||
retry = 2
|
retry = 2
|
||||||
while retry > 0:
|
while retry > 0:
|
||||||
retry -= 1
|
retry -= 1
|
||||||
try:
|
try:
|
||||||
return await super().arequest(method, url, headers, stream, ext)
|
return await super().handle_async_request(method, url, headers, stream, extensions)
|
||||||
except (python_socks._errors.ProxyConnectionError,
|
except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e:
|
||||||
python_socks._errors.ProxyTimeoutError,
|
raise httpx.ProxyError(e)
|
||||||
python_socks._errors.ProxyError) as e:
|
|
||||||
raise httpcore.ProxyError(e)
|
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
# socket.gaierror when DNS resolution fails
|
# socket.gaierror when DNS resolution fails
|
||||||
raise httpcore.NetworkError(e)
|
raise httpx.NetworkError(e)
|
||||||
except httpcore.RemoteProtocolError as e:
|
except httpx.RemoteProtocolError as e:
|
||||||
# in case of httpcore.RemoteProtocolError: Server disconnected
|
# in case of httpx.RemoteProtocolError: Server disconnected
|
||||||
await close_connections_for_url(self, url)
|
await close_connections_for_url(self, url)
|
||||||
logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
|
logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
|
||||||
# retry
|
# retry
|
||||||
except (httpcore.NetworkError, httpcore.ProtocolError) as e:
|
except (httpx.NetworkError, httpx.ProtocolError) as e:
|
||||||
# httpcore.WriteError on HTTP/2 connection leaves a new opened stream
|
# httpx.WriteError on HTTP/2 connection leaves a new opened stream
|
||||||
# then each new request creates a new stream and raise the same WriteError
|
# then each new request creates a new stream and raise the same WriteError
|
||||||
await close_connections_for_url(self, url)
|
await close_connections_for_url(self, url)
|
||||||
raise e
|
raise e
|
||||||
|
@ -102,28 +102,28 @@ class AsyncProxyTransportFixed(AsyncProxyTransport):
|
||||||
class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
|
class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
|
||||||
"""Fix httpx.AsyncHTTPTransport"""
|
"""Fix httpx.AsyncHTTPTransport"""
|
||||||
|
|
||||||
async def arequest(self, method, url, headers=None, stream=None, ext=None):
|
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None):
|
||||||
retry = 2
|
retry = 2
|
||||||
while retry > 0:
|
while retry > 0:
|
||||||
retry -= 1
|
retry -= 1
|
||||||
try:
|
try:
|
||||||
return await super().arequest(method, url, headers, stream, ext)
|
return await super().handle_async_request(method, url, headers, stream, extensions)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
# socket.gaierror when DNS resolution fails
|
# socket.gaierror when DNS resolution fails
|
||||||
raise httpcore.ConnectError(e)
|
raise httpx.ConnectError(e)
|
||||||
except httpcore.CloseError as e:
|
except httpx.CloseError as e:
|
||||||
# httpcore.CloseError: [Errno 104] Connection reset by peer
|
# httpx.CloseError: [Errno 104] Connection reset by peer
|
||||||
# raised by _keepalive_sweep()
|
# raised by _keepalive_sweep()
|
||||||
# from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # noqa
|
# from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # noqa
|
||||||
await close_connections_for_url(self._pool, url)
|
await close_connections_for_url(self._pool, url)
|
||||||
logger.warning('httpcore.CloseError: retry', exc_info=e)
|
logger.warning('httpx.CloseError: retry', exc_info=e)
|
||||||
# retry
|
# retry
|
||||||
except httpcore.RemoteProtocolError as e:
|
except httpx.RemoteProtocolError as e:
|
||||||
# in case of httpcore.RemoteProtocolError: Server disconnected
|
# in case of httpx.RemoteProtocolError: Server disconnected
|
||||||
await close_connections_for_url(self._pool, url)
|
await close_connections_for_url(self._pool, url)
|
||||||
logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
|
logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
|
||||||
# retry
|
# retry
|
||||||
except (httpcore.ProtocolError, httpcore.NetworkError) as e:
|
except (httpx.ProtocolError, httpx.NetworkError) as e:
|
||||||
await close_connections_for_url(self._pool, url)
|
await close_connections_for_url(self._pool, url)
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue