[enh] add searx.shared

shared dictionary between the workers (UWSGI or werkzeug)
scheduler: run a task once every x seconds (UWSGI or werkzeug)
This commit is contained in:
Alexandre Flament 2021-01-05 11:22:48 +01:00
parent 9c581466e1
commit 6e2872f436
7 changed files with 156 additions and 1 deletions

View File

@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
static-expires = /* 864000
static-gzip-all = True
offload-threads = %k
# Cache
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1

31
searx/shared/__init__.py Normal file
View File

@ -0,0 +1,31 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
import logging
logger = logging.getLogger('searx.shared')
try:
import uwsgi
except:
# no uwsgi
from .shared_simple import SimpleSharedDict as SharedDict, schedule
logger.info('Use shared_simple implementation')
else:
try:
uwsgi.cache_update('dummy', b'dummy')
if uwsgi.cache_get('dummy') != b'dummy':
raise Exception()
except:
# uwsgi.ini configuration problem: disable all scheduling
logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
from .shared_simple import SimpleSharedDict as SharedDict
def schedule(delay, func, *args):
pass
else:
# uwsgi
from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
logger.info('Use shared_uwsgi implementation')
storage = SharedDict()

View File

@ -0,0 +1,15 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
class SharedDict:
def get_int(self, key):
pass
def set_int(self, key, value):
pass
def get_str(self, key):
pass
def set_str(self, key, value):
pass

View File

@ -0,0 +1,38 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
import threading
from . import shared_abstract
class SimpleSharedDict(shared_abstract.SharedDict):
__slots__ = 'd',
def __init__(self):
self.d = {}
def get_int(self, key):
return self.d.get(key, None)
def set_int(self, key, value):
self.d[key] = value
def get_str(self, key):
return self.d.get(key, None)
def set_str(self, key, value):
self.d[key] = value
def schedule(delay, func, *args):
def call_later():
t = threading.Timer(delay, wrapper)
t.daemon = True
t.start()
def wrapper():
call_later()
func(*args)
call_later()

View File

@ -0,0 +1,62 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
import time
import uwsgi # pylint: disable=E0401
from . import shared_abstract
_last_signal = 10
class UwsgiCacheSharedDict(shared_abstract.SharedDict):
def get_int(self, key):
value = uwsgi.cache_get(key)
if value is None:
return value
else:
return int.from_bytes(value, 'big')
def set_int(self, key, value):
b = value.to_bytes(4, 'big')
uwsgi.cache_update(key, b)
def get_str(self, key):
value = uwsgi.cache_get(key)
if value is None:
return value
else:
return value.decode('utf-8')
def set_str(self, key, value):
b = value.encode('utf-8')
uwsgi.cache_update(key, b)
def schedule(delay, func, *args):
"""
Can be implemented using a spooler.
https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
To make the uwsgi configuration simple, use the alternative implementation.
"""
global _last_signal
def sighandler(signum):
now = int(time.time())
uwsgi.lock()
try:
updating = uwsgi.cache_get('updating')
if updating is not None:
updating = int.from_bytes(updating, 'big')
if now - updating < delay:
return
uwsgi.cache_update('updating', now.to_bytes(4, 'big'))
finally:
uwsgi.unlock()
func(*args)
signal_num = _last_signal
_last_signal += 1
uwsgi.register_signal(signal_num, 'worker', sighandler)
uwsgi.add_timer(signal_num, delay)

View File

@ -83,3 +83,6 @@ http = ${SEARX_INTERNAL_HTTP}
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket
# Cache
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1

View File

@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP}
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket
# Cache
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1