mirror of
https://github.com/searx/searx
synced 2024-12-14 21:30:01 +01:00
[enh] support of multiple outgoing IPs
This commit is contained in:
parent
7f7f10bb6f
commit
03579c225f
@ -1,20 +1,63 @@
|
||||
import requests
|
||||
from itertools import cycle
|
||||
from searx import settings
|
||||
|
||||
|
||||
the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
|
||||
the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
|
||||
class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
|
||||
|
||||
def __init__(self, pool_connections=requests.adapters.DEFAULT_POOLSIZE,
|
||||
pool_maxsize=requests.adapters.DEFAULT_POOLSIZE,
|
||||
max_retries=requests.adapters.DEFAULT_RETRIES,
|
||||
pool_block=requests.adapters.DEFAULT_POOLBLOCK,
|
||||
**conn_params):
|
||||
if max_retries == requests.adapters.DEFAULT_RETRIES:
|
||||
self.max_retries = requests.adapters.Retry(0, read=False)
|
||||
else:
|
||||
self.max_retries = requests.adapters.Retry.from_int(max_retries)
|
||||
self.config = {}
|
||||
self.proxy_manager = {}
|
||||
|
||||
super(requests.adapters.HTTPAdapter, self).__init__()
|
||||
|
||||
self._pool_connections = pool_connections
|
||||
self._pool_maxsize = pool_maxsize
|
||||
self._pool_block = pool_block
|
||||
self._conn_params = conn_params
|
||||
|
||||
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block, **conn_params)
|
||||
|
||||
def __setstate__(self, state):
|
||||
# Can't handle by adding 'proxy_manager' to self.__attrs__ because
|
||||
# because self.poolmanager uses a lambda function, which isn't pickleable.
|
||||
self.proxy_manager = {}
|
||||
self.config = {}
|
||||
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.init_poolmanager(self._pool_connections, self._pool_maxsize,
|
||||
block=self._pool_block, **self._conn_params)
|
||||
|
||||
|
||||
if settings.get('source_ips'):
|
||||
http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
|
||||
for source_ip in settings['source_ips'])
|
||||
https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
|
||||
for source_ip in settings['source_ips'])
|
||||
else:
|
||||
http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
|
||||
https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
|
||||
|
||||
|
||||
class SessionSinglePool(requests.Session):
|
||||
|
||||
def __init__(self):
|
||||
global the_https_adapter, the_http_adapter
|
||||
super(SessionSinglePool, self).__init__()
|
||||
|
||||
# reuse the same adapters
|
||||
self.adapters.clear()
|
||||
self.mount('https://', the_https_adapter)
|
||||
self.mount('http://', the_http_adapter)
|
||||
self.mount('https://', next(https_adapters))
|
||||
self.mount('http://', next(http_adapters))
|
||||
|
||||
def close(self):
|
||||
"""Call super, but clear adapters since there are managed globaly"""
|
||||
|
@ -11,6 +11,12 @@ server:
|
||||
image_proxy : False # Proxying image results through searx
|
||||
default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
|
||||
|
||||
# uncomment below section only if you have more than one network interface
|
||||
# which can be the source of outgoing search requests
|
||||
#source_ips:
|
||||
# - 1.1.1.1
|
||||
# - 1.1.1.2
|
||||
|
||||
engines:
|
||||
- name : wikipedia
|
||||
engine : mediawiki
|
||||
|
Loading…
Reference in New Issue
Block a user