diff --git a/searx/search/__init__.py b/searx/search/__init__.py index 93043e40..f2f774bb 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -25,13 +25,12 @@ from _thread import start_new_thread from searx import settings from searx.answerers import ask from searx.external_bang import get_bang_url -from searx.engines import load_engines from searx.results import ResultContainer from searx import logger from searx.plugins import plugins from searx.search.models import EngineRef, SearchQuery -from searx.search.processors import PROCESSORS, initialize as initialize_processors -from searx.network import check_network_configuration, initialize as initialize_network +from searx.search.processors import processors, initialize as initialize_processors +from searx.network import check_network_configuration from searx.search.checker import initialize as initialize_checker @@ -51,12 +50,9 @@ else: def initialize(settings_engines=None, enable_checker=False, check_network=False): settings_engines = settings_engines or settings['engines'] - load_engines(settings_engines) - initialize_network(settings_engines, settings['outgoing']) + initialize_processors(settings_engines) if check_network: check_network_configuration() - initialize_processors(settings_engines) - if enable_checker: initialize_checker() @@ -111,7 +107,7 @@ class Search: # start search-reqest for all selected engines for engineref in self.search_query.engineref_list: - processor = PROCESSORS[engineref.name] + processor = processors[engineref.name] # set default request parameters request_params = processor.get_params(self.search_query, engineref.category) @@ -154,7 +150,7 @@ class Search: for engine_name, query, request_params in requests: th = threading.Thread( - target=PROCESSORS[engine_name].search, + target=processors[engine_name].search, args=(query, request_params, self.result_container, self.start_time, self.actual_timeout), name=search_id, ) diff --git a/searx/search/checker/background.py b/searx/search/checker/background.py index 276426fa..c3292d9a 100644 --- a/searx/search/checker/background.py +++ b/searx/search/checker/background.py @@ -9,7 +9,7 @@ import signal from searx import logger, settings, searx_debug from searx.exceptions import SearxSettingsException -from searx.search.processors import PROCESSORS +from searx.search.processors import processors from searx.search.checker import Checker from searx.shared import schedule, storage @@ -55,7 +55,7 @@ def run(): 'status': 'ok', 'engines': {} } - for name, processor in PROCESSORS.items(): + for name, processor in processors.items(): logger.debug('Checking %s engine', name) checker = Checker(processor) checker.run() diff --git a/searx/search/processors/__init__.py b/searx/search/processors/__init__.py index c2f6df17..4cae3cd0 100644 --- a/searx/search/processors/__init__.py +++ b/searx/search/processors/__init__.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -import threading - from .online import OnlineProcessor from .offline import OfflineProcessor from .online_dictionary import OnlineDictionaryProcessor @@ -12,9 +10,9 @@ import searx.engines as engines __all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor', - 'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'PROCESSORS'] + 'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'processors'] logger = logger.getChild('search.processors') -PROCESSORS = {} +processors = {} def get_processor_class(engine_type): @@ -29,27 +27,15 @@ def get_processor(engine, engine_name): processor_class = get_processor_class(engine_type) if processor_class: return processor_class(engine, engine_name) - return None - - -def initialize_processor(processor): - """Initialize one processor - Call the init function of the engine - """ - if processor.has_initialize_function: - t = threading.Thread(target=processor.initialize, daemon=True) - t.start() + else: + return None def initialize(engine_list): - """Initialize all engines and store a processor for each engine in :py:obj:`PROCESSORS`.""" - for engine_data in engine_list: - engine_name = engine_data['name'] - engine = engines.engines.get(engine_name) - if engine: - processor = get_processor(engine, engine_name) - initialize_processor(processor) - if processor is None: - engine.logger.error('Error get processor for engine %s', engine_name) - else: - PROCESSORS[engine_name] = processor + engines.initialize_engines(engine_list) + for engine_name, engine in engines.engines.items(): + processor = get_processor(engine, engine_name) + if processor is None: + logger.error('Error get processor for engine %s', engine_name) + else: + processors[engine_name] = processor diff --git a/searx/search/processors/abstract.py b/searx/search/processors/abstract.py index e676b1f5..26dab069 100644 --- a/searx/search/processors/abstract.py +++ b/searx/search/processors/abstract.py @@ -2,32 +2,17 @@ from abc import abstractmethod, ABC from searx import logger -from searx.engines import engines -from searx.utils import get_engine_from_settings logger = logger.getChild('searx.search.processor') class EngineProcessor(ABC): + def __init__(self, engine, engine_name): self.engine = engine self.engine_name = engine_name - def initialize(self): - try: - self.engine.init(get_engine_from_settings(self.engine_name)) - except SearxEngineResponseException as exc: - logger.warn('Fail to initialize %s // %s', self.engine_name, exc) - except Exception: # pylint: disable=broad-except - logger.exception('Fail to initialize %s', self.engine_name) - else: - logger.debug('Initialized %s', self.engine_name) - - @property - def has_initialize_function(self): - return hasattr(self.engine, 'init') - def get_params(self, search_query, engine_category): # if paging is not supported, skip if search_query.pageno > 1 and not self.engine.paging: diff --git a/tests/unit/test_engines_init.py b/tests/unit/test_engines_init.py index d2aee41f..c75637f2 100644 --- a/tests/unit/test_engines_init.py +++ b/tests/unit/test_engines_init.py @@ -23,7 +23,7 @@ class TestEnginesInit(SearxTestCase): engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general'}, {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}] - engines.load_engines(engine_list) + engines.initialize_engines(engine_list) self.assertEqual(len(engines.engines), 1) self.assertIn('engine1', engines.engines) self.assertNotIn('onions', engines.categories) @@ -35,7 +35,7 @@ class TestEnginesInit(SearxTestCase): 'timeout': 20.0, 'onion_url': 'http://engine1.onion'}, {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}] - engines.load_engines(engine_list) + engines.initialize_engines(engine_list) self.assertEqual(len(engines.engines), 2) self.assertIn('engine1', engines.engines) self.assertIn('engine2', engines.engines) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 83be9ef3..7a79ce24 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1,9 +1,11 @@ -from searx import settings -from searx.engines import load_engines +from mock import patch + from searx.search import initialize from searx.query import RawTextQuery from searx.testing import SearxTestCase +import searx.engines + TEST_ENGINES = [ { @@ -279,6 +281,10 @@ class TestBang(SearxTestCase): self.assertEqual(query.getQuery(), '!dum the query') def test_bang_autocomplete_empty(self): - load_engines(settings['engines']) - query = RawTextQuery('the query !', []) - self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm']) + with patch.object(searx.engines, 'initialize_engines', searx.engines.load_engines): + initialize() + query = RawTextQuery('the query !', []) + self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm']) + + query = RawTextQuery('the query ?', ['osm']) + self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia']) diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index f865ef03..9c598a16 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -3,20 +3,16 @@ import json from urllib.parse import ParseResult from mock import Mock - -import searx.search.processors from searx.testing import SearxTestCase from searx.search import Search +import searx.engines class ViewsTestCase(SearxTestCase): def setUp(self): # skip init function (no external HTTP request) - def dummy(*args, **kwargs): - pass - - self.setattr4test(searx.search.processors, 'initialize_processor', dummy) + self.setattr4test(searx.engines, 'initialize_engines', searx.engines.load_engines) from searx import webapp # pylint disable=import-outside-toplevel