mirror of https://github.com/searx/searx
Revert "Fix issues in network after updating httpx to 0.21.x (#3169)"
This reverts commit 0669bfd7a5
.
This commit is contained in:
parent
c8143a98d8
commit
52351b044f
|
@ -25,13 +25,12 @@ from _thread import start_new_thread
|
||||||
from searx import settings
|
from searx import settings
|
||||||
from searx.answerers import ask
|
from searx.answerers import ask
|
||||||
from searx.external_bang import get_bang_url
|
from searx.external_bang import get_bang_url
|
||||||
from searx.engines import load_engines
|
|
||||||
from searx.results import ResultContainer
|
from searx.results import ResultContainer
|
||||||
from searx import logger
|
from searx import logger
|
||||||
from searx.plugins import plugins
|
from searx.plugins import plugins
|
||||||
from searx.search.models import EngineRef, SearchQuery
|
from searx.search.models import EngineRef, SearchQuery
|
||||||
from searx.search.processors import PROCESSORS, initialize as initialize_processors
|
from searx.search.processors import processors, initialize as initialize_processors
|
||||||
from searx.network import check_network_configuration, initialize as initialize_network
|
from searx.network import check_network_configuration
|
||||||
from searx.search.checker import initialize as initialize_checker
|
from searx.search.checker import initialize as initialize_checker
|
||||||
|
|
||||||
|
|
||||||
|
@ -51,12 +50,9 @@ else:
|
||||||
|
|
||||||
def initialize(settings_engines=None, enable_checker=False, check_network=False):
|
def initialize(settings_engines=None, enable_checker=False, check_network=False):
|
||||||
settings_engines = settings_engines or settings['engines']
|
settings_engines = settings_engines or settings['engines']
|
||||||
load_engines(settings_engines)
|
initialize_processors(settings_engines)
|
||||||
initialize_network(settings_engines, settings['outgoing'])
|
|
||||||
if check_network:
|
if check_network:
|
||||||
check_network_configuration()
|
check_network_configuration()
|
||||||
initialize_processors(settings_engines)
|
|
||||||
|
|
||||||
if enable_checker:
|
if enable_checker:
|
||||||
initialize_checker()
|
initialize_checker()
|
||||||
|
|
||||||
|
@ -111,7 +107,7 @@ class Search:
|
||||||
|
|
||||||
# start search-reqest for all selected engines
|
# start search-reqest for all selected engines
|
||||||
for engineref in self.search_query.engineref_list:
|
for engineref in self.search_query.engineref_list:
|
||||||
processor = PROCESSORS[engineref.name]
|
processor = processors[engineref.name]
|
||||||
|
|
||||||
# set default request parameters
|
# set default request parameters
|
||||||
request_params = processor.get_params(self.search_query, engineref.category)
|
request_params = processor.get_params(self.search_query, engineref.category)
|
||||||
|
@ -154,7 +150,7 @@ class Search:
|
||||||
|
|
||||||
for engine_name, query, request_params in requests:
|
for engine_name, query, request_params in requests:
|
||||||
th = threading.Thread(
|
th = threading.Thread(
|
||||||
target=PROCESSORS[engine_name].search,
|
target=processors[engine_name].search,
|
||||||
args=(query, request_params, self.result_container, self.start_time, self.actual_timeout),
|
args=(query, request_params, self.result_container, self.start_time, self.actual_timeout),
|
||||||
name=search_id,
|
name=search_id,
|
||||||
)
|
)
|
||||||
|
|
|
@ -9,7 +9,7 @@ import signal
|
||||||
|
|
||||||
from searx import logger, settings, searx_debug
|
from searx import logger, settings, searx_debug
|
||||||
from searx.exceptions import SearxSettingsException
|
from searx.exceptions import SearxSettingsException
|
||||||
from searx.search.processors import PROCESSORS
|
from searx.search.processors import processors
|
||||||
from searx.search.checker import Checker
|
from searx.search.checker import Checker
|
||||||
from searx.shared import schedule, storage
|
from searx.shared import schedule, storage
|
||||||
|
|
||||||
|
@ -55,7 +55,7 @@ def run():
|
||||||
'status': 'ok',
|
'status': 'ok',
|
||||||
'engines': {}
|
'engines': {}
|
||||||
}
|
}
|
||||||
for name, processor in PROCESSORS.items():
|
for name, processor in processors.items():
|
||||||
logger.debug('Checking %s engine', name)
|
logger.debug('Checking %s engine', name)
|
||||||
checker = Checker(processor)
|
checker = Checker(processor)
|
||||||
checker.run()
|
checker.run()
|
||||||
|
|
|
@ -1,7 +1,5 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
import threading
|
|
||||||
|
|
||||||
from .online import OnlineProcessor
|
from .online import OnlineProcessor
|
||||||
from .offline import OfflineProcessor
|
from .offline import OfflineProcessor
|
||||||
from .online_dictionary import OnlineDictionaryProcessor
|
from .online_dictionary import OnlineDictionaryProcessor
|
||||||
|
@ -12,9 +10,9 @@ import searx.engines as engines
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor',
|
__all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor',
|
||||||
'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'PROCESSORS']
|
'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'processors']
|
||||||
logger = logger.getChild('search.processors')
|
logger = logger.getChild('search.processors')
|
||||||
PROCESSORS = {}
|
processors = {}
|
||||||
|
|
||||||
|
|
||||||
def get_processor_class(engine_type):
|
def get_processor_class(engine_type):
|
||||||
|
@ -29,27 +27,15 @@ def get_processor(engine, engine_name):
|
||||||
processor_class = get_processor_class(engine_type)
|
processor_class = get_processor_class(engine_type)
|
||||||
if processor_class:
|
if processor_class:
|
||||||
return processor_class(engine, engine_name)
|
return processor_class(engine, engine_name)
|
||||||
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def initialize_processor(processor):
|
|
||||||
"""Initialize one processor
|
|
||||||
Call the init function of the engine
|
|
||||||
"""
|
|
||||||
if processor.has_initialize_function:
|
|
||||||
t = threading.Thread(target=processor.initialize, daemon=True)
|
|
||||||
t.start()
|
|
||||||
|
|
||||||
|
|
||||||
def initialize(engine_list):
|
def initialize(engine_list):
|
||||||
"""Initialize all engines and store a processor for each engine in :py:obj:`PROCESSORS`."""
|
engines.initialize_engines(engine_list)
|
||||||
for engine_data in engine_list:
|
for engine_name, engine in engines.engines.items():
|
||||||
engine_name = engine_data['name']
|
|
||||||
engine = engines.engines.get(engine_name)
|
|
||||||
if engine:
|
|
||||||
processor = get_processor(engine, engine_name)
|
processor = get_processor(engine, engine_name)
|
||||||
initialize_processor(processor)
|
|
||||||
if processor is None:
|
if processor is None:
|
||||||
engine.logger.error('Error get processor for engine %s', engine_name)
|
logger.error('Error get processor for engine %s', engine_name)
|
||||||
else:
|
else:
|
||||||
PROCESSORS[engine_name] = processor
|
processors[engine_name] = processor
|
||||||
|
|
|
@ -2,32 +2,17 @@
|
||||||
|
|
||||||
from abc import abstractmethod, ABC
|
from abc import abstractmethod, ABC
|
||||||
from searx import logger
|
from searx import logger
|
||||||
from searx.engines import engines
|
|
||||||
from searx.utils import get_engine_from_settings
|
|
||||||
|
|
||||||
|
|
||||||
logger = logger.getChild('searx.search.processor')
|
logger = logger.getChild('searx.search.processor')
|
||||||
|
|
||||||
|
|
||||||
class EngineProcessor(ABC):
|
class EngineProcessor(ABC):
|
||||||
|
|
||||||
def __init__(self, engine, engine_name):
|
def __init__(self, engine, engine_name):
|
||||||
self.engine = engine
|
self.engine = engine
|
||||||
self.engine_name = engine_name
|
self.engine_name = engine_name
|
||||||
|
|
||||||
def initialize(self):
|
|
||||||
try:
|
|
||||||
self.engine.init(get_engine_from_settings(self.engine_name))
|
|
||||||
except SearxEngineResponseException as exc:
|
|
||||||
logger.warn('Fail to initialize %s // %s', self.engine_name, exc)
|
|
||||||
except Exception: # pylint: disable=broad-except
|
|
||||||
logger.exception('Fail to initialize %s', self.engine_name)
|
|
||||||
else:
|
|
||||||
logger.debug('Initialized %s', self.engine_name)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def has_initialize_function(self):
|
|
||||||
return hasattr(self.engine, 'init')
|
|
||||||
|
|
||||||
def get_params(self, search_query, engine_category):
|
def get_params(self, search_query, engine_category):
|
||||||
# if paging is not supported, skip
|
# if paging is not supported, skip
|
||||||
if search_query.pageno > 1 and not self.engine.paging:
|
if search_query.pageno > 1 and not self.engine.paging:
|
||||||
|
|
|
@ -23,7 +23,7 @@ class TestEnginesInit(SearxTestCase):
|
||||||
engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general'},
|
engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general'},
|
||||||
{'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
|
{'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
|
||||||
|
|
||||||
engines.load_engines(engine_list)
|
engines.initialize_engines(engine_list)
|
||||||
self.assertEqual(len(engines.engines), 1)
|
self.assertEqual(len(engines.engines), 1)
|
||||||
self.assertIn('engine1', engines.engines)
|
self.assertIn('engine1', engines.engines)
|
||||||
self.assertNotIn('onions', engines.categories)
|
self.assertNotIn('onions', engines.categories)
|
||||||
|
@ -35,7 +35,7 @@ class TestEnginesInit(SearxTestCase):
|
||||||
'timeout': 20.0, 'onion_url': 'http://engine1.onion'},
|
'timeout': 20.0, 'onion_url': 'http://engine1.onion'},
|
||||||
{'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
|
{'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
|
||||||
|
|
||||||
engines.load_engines(engine_list)
|
engines.initialize_engines(engine_list)
|
||||||
self.assertEqual(len(engines.engines), 2)
|
self.assertEqual(len(engines.engines), 2)
|
||||||
self.assertIn('engine1', engines.engines)
|
self.assertIn('engine1', engines.engines)
|
||||||
self.assertIn('engine2', engines.engines)
|
self.assertIn('engine2', engines.engines)
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
from searx import settings
|
from mock import patch
|
||||||
from searx.engines import load_engines
|
|
||||||
from searx.search import initialize
|
from searx.search import initialize
|
||||||
from searx.query import RawTextQuery
|
from searx.query import RawTextQuery
|
||||||
from searx.testing import SearxTestCase
|
from searx.testing import SearxTestCase
|
||||||
|
|
||||||
|
import searx.engines
|
||||||
|
|
||||||
|
|
||||||
TEST_ENGINES = [
|
TEST_ENGINES = [
|
||||||
{
|
{
|
||||||
|
@ -279,6 +281,10 @@ class TestBang(SearxTestCase):
|
||||||
self.assertEqual(query.getQuery(), '!dum the query')
|
self.assertEqual(query.getQuery(), '!dum the query')
|
||||||
|
|
||||||
def test_bang_autocomplete_empty(self):
|
def test_bang_autocomplete_empty(self):
|
||||||
load_engines(settings['engines'])
|
with patch.object(searx.engines, 'initialize_engines', searx.engines.load_engines):
|
||||||
|
initialize()
|
||||||
query = RawTextQuery('the query !', [])
|
query = RawTextQuery('the query !', [])
|
||||||
self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
|
self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
|
||||||
|
|
||||||
|
query = RawTextQuery('the query ?', ['osm'])
|
||||||
|
self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])
|
||||||
|
|
|
@ -3,20 +3,16 @@
|
||||||
import json
|
import json
|
||||||
from urllib.parse import ParseResult
|
from urllib.parse import ParseResult
|
||||||
from mock import Mock
|
from mock import Mock
|
||||||
|
|
||||||
import searx.search.processors
|
|
||||||
from searx.testing import SearxTestCase
|
from searx.testing import SearxTestCase
|
||||||
from searx.search import Search
|
from searx.search import Search
|
||||||
|
import searx.engines
|
||||||
|
|
||||||
|
|
||||||
class ViewsTestCase(SearxTestCase):
|
class ViewsTestCase(SearxTestCase):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
# skip init function (no external HTTP request)
|
# skip init function (no external HTTP request)
|
||||||
def dummy(*args, **kwargs):
|
self.setattr4test(searx.engines, 'initialize_engines', searx.engines.load_engines)
|
||||||
pass
|
|
||||||
|
|
||||||
self.setattr4test(searx.search.processors, 'initialize_processor', dummy)
|
|
||||||
|
|
||||||
from searx import webapp # pylint disable=import-outside-toplevel
|
from searx import webapp # pylint disable=import-outside-toplevel
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue