Revert "Fix issues in network after updating httpx to 0.21.x (#3169)"

This reverts commit 0669bfd7a5.

searx/search/__init__.py

@@ -25,13 +25,12 @@ from _thread import start_new_thread
 from searx import settings
 from searx.answerers import ask
 from searx.external_bang import get_bang_url
-from searx.engines import load_engines
 from searx.results import ResultContainer
 from searx import logger
 from searx.plugins import plugins
 from searx.search.models import EngineRef, SearchQuery
-from searx.search.processors import PROCESSORS, initialize as initialize_processors
-from searx.network import check_network_configuration, initialize as initialize_network
+from searx.search.processors import processors, initialize as initialize_processors
+from searx.network import check_network_configuration
 from searx.search.checker import initialize as initialize_checker
 
 
@@ -51,12 +50,9 @@ else:
 
 def initialize(settings_engines=None, enable_checker=False, check_network=False):
     settings_engines = settings_engines or settings['engines']
-    load_engines(settings_engines)
-    initialize_network(settings_engines, settings['outgoing'])
+    initialize_processors(settings_engines)
     if check_network:
         check_network_configuration()
-    initialize_processors(settings_engines)
-
     if enable_checker:
         initialize_checker()
 
@@ -111,7 +107,7 @@ class Search:
 
         # start search-reqest for all selected engines
         for engineref in self.search_query.engineref_list:
-            processor = PROCESSORS[engineref.name]
+            processor = processors[engineref.name]
 
             # set default request parameters
             request_params = processor.get_params(self.search_query, engineref.category)
@@ -154,7 +150,7 @@ class Search:
 
         for engine_name, query, request_params in requests:
             th = threading.Thread(
-                target=PROCESSORS[engine_name].search,
+                target=processors[engine_name].search,
                 args=(query, request_params, self.result_container, self.start_time, self.actual_timeout),
                 name=search_id,
             )

searx/search/checker/background.py

@@ -9,7 +9,7 @@ import signal
 
 from searx import logger, settings, searx_debug
 from searx.exceptions import SearxSettingsException
-from searx.search.processors import PROCESSORS
+from searx.search.processors import processors
 from searx.search.checker import Checker
 from searx.shared import schedule, storage
 
@@ -55,7 +55,7 @@ def run():
             'status': 'ok',
             'engines': {}
         }
-        for name, processor in PROCESSORS.items():
+        for name, processor in processors.items():
             logger.debug('Checking %s engine', name)
             checker = Checker(processor)
             checker.run()

searx/search/processors/__init__.py

@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-import threading
-
 from .online import OnlineProcessor
 from .offline import OfflineProcessor
 from .online_dictionary import OnlineDictionaryProcessor
@@ -12,9 +10,9 @@ import searx.engines as engines
 
 
 __all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor',
-           'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'PROCESSORS']
+           'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'processors']
 logger = logger.getChild('search.processors')
-PROCESSORS = {}
+processors = {}
 
 
 def get_processor_class(engine_type):
@@ -29,27 +27,15 @@ def get_processor(engine, engine_name):
     processor_class = get_processor_class(engine_type)
     if processor_class:
         return processor_class(engine, engine_name)
-    return None
-
-
-def initialize_processor(processor):
-    """Initialize one processor
-    Call the init function of the engine
-    """
-    if processor.has_initialize_function:
-        t = threading.Thread(target=processor.initialize, daemon=True)
-        t.start()
+    else:
+        return None
 
 
 def initialize(engine_list):
-    """Initialize all engines and store a processor for each engine in :py:obj:`PROCESSORS`."""
-    for engine_data in engine_list:
-        engine_name = engine_data['name']
-        engine = engines.engines.get(engine_name)
-        if engine:
-            processor = get_processor(engine, engine_name)
-            initialize_processor(processor)
-            if processor is None:
-                engine.logger.error('Error get processor for engine %s', engine_name)
-            else:
-                PROCESSORS[engine_name] = processor
+    engines.initialize_engines(engine_list)
+    for engine_name, engine in engines.engines.items():
+        processor = get_processor(engine, engine_name)
+        if processor is None:
+            logger.error('Error get processor for engine %s', engine_name)
+        else:
+            processors[engine_name] = processor

searx/search/processors/abstract.py

@@ -2,32 +2,17 @@
 
 from abc import abstractmethod, ABC
 from searx import logger
-from searx.engines import engines
-from searx.utils import get_engine_from_settings
 
 
 logger = logger.getChild('searx.search.processor')
 
 
 class EngineProcessor(ABC):
+
     def __init__(self, engine, engine_name):
         self.engine = engine
         self.engine_name = engine_name
 
-    def initialize(self):
-        try:
-            self.engine.init(get_engine_from_settings(self.engine_name))
-        except SearxEngineResponseException as exc:
-            logger.warn('Fail to initialize %s // %s', self.engine_name, exc)
-        except Exception:  # pylint: disable=broad-except
-            logger.exception('Fail to initialize %s', self.engine_name)
-        else:
-            logger.debug('Initialized %s', self.engine_name)
-
-    @property
-    def has_initialize_function(self):
-        return hasattr(self.engine, 'init')
-
     def get_params(self, search_query, engine_category):
         # if paging is not supported, skip
         if search_query.pageno > 1 and not self.engine.paging:

tests/unit/test_engines_init.py

@@ -23,7 +23,7 @@ class TestEnginesInit(SearxTestCase):
         engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general'},
                        {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
 
-        engines.load_engines(engine_list)
+        engines.initialize_engines(engine_list)
         self.assertEqual(len(engines.engines), 1)
         self.assertIn('engine1', engines.engines)
         self.assertNotIn('onions', engines.categories)
@@ -35,7 +35,7 @@ class TestEnginesInit(SearxTestCase):
                         'timeout': 20.0, 'onion_url': 'http://engine1.onion'},
                        {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]
 
-        engines.load_engines(engine_list)
+        engines.initialize_engines(engine_list)
         self.assertEqual(len(engines.engines), 2)
         self.assertIn('engine1', engines.engines)
         self.assertIn('engine2', engines.engines)

tests/unit/test_query.py

@@ -1,9 +1,11 @@
-from searx import settings
-from searx.engines import load_engines
+from mock import patch
+
 from searx.search import initialize
 from searx.query import RawTextQuery
 from searx.testing import SearxTestCase
 
+import searx.engines
+
 
 TEST_ENGINES = [
     {
@@ -279,6 +281,10 @@ class TestBang(SearxTestCase):
         self.assertEqual(query.getQuery(), '!dum the query')
 
     def test_bang_autocomplete_empty(self):
-        load_engines(settings['engines'])
-        query = RawTextQuery('the query !', [])
-        self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
+        with patch.object(searx.engines, 'initialize_engines', searx.engines.load_engines):
+            initialize()
+            query = RawTextQuery('the query !', [])
+            self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
+
+            query = RawTextQuery('the query ?', ['osm'])
+            self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])

tests/unit/test_webapp.py

@@ -3,20 +3,16 @@
 import json
 from urllib.parse import ParseResult
 from mock import Mock
-
-import searx.search.processors
 from searx.testing import SearxTestCase
 from searx.search import Search
+import searx.engines
 
 
 class ViewsTestCase(SearxTestCase):
 
     def setUp(self):
         # skip init function (no external HTTP request)
-        def dummy(*args, **kwargs):
-            pass
-
-        self.setattr4test(searx.search.processors, 'initialize_processor', dummy)
+        self.setattr4test(searx.engines, 'initialize_engines', searx.engines.load_engines)
 
         from searx import webapp  # pylint disable=import-outside-toplevel