Add searx_extra package

Split the utils directory into:
* searx_extra contains update scripts, standalone_searx.py
* utils contains the files to build and setup searx.
This commit is contained in:
Alexandre Flament 2021-02-25 17:42:52 +01:00
parent 111d38cd8f
commit b8cd326464
21 changed files with 47 additions and 75 deletions

View File

@ -11,12 +11,12 @@ jobs:
strategy: strategy:
matrix: matrix:
fetch: fetch:
- ahmia_blacklist - update_ahmia_blacklist.py
- currencies - update_currencies.py
- external_bangs - update_external_bangs.py
- firefox_version - update_firefox_version.py
- languages - update_languages.py
- wikidata_units - update_wikidata_units.py
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -45,10 +45,10 @@ jobs:
- name: Fetch data - name: Fetch data
env: env:
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
run: | run: |
source local/py3/bin/activate source local/py3/bin/activate
python $FETCH_SCRIPT $FETCH_SCRIPT
- name: Create Pull Request - name: Create Pull Request
id: cpr id: cpr

View File

@ -195,8 +195,8 @@ PYLINT_FILES=\
searx/engines/google_videos.py \ searx/engines/google_videos.py \
searx/engines/google_images.py \ searx/engines/google_images.py \
searx/engines/mediathekviewweb.py \ searx/engines/mediathekviewweb.py \
utils/fetch_external_bangs.py \ searx/engines/google_scholar.py \
searx/engines/google_scholar.py searx_extra/update/update_external_bangs.py
test.pylint: pyenvinstall test.pylint: pyenvinstall
$(call cmd,pylint,$(PYLINT_FILES)) $(call cmd,pylint,$(PYLINT_FILES))

View File

@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
user/index user/index
admin/index admin/index
dev/index dev/index
searx_extra/index
utils/index utils/index
blog/index blog/index

View File

@ -0,0 +1,14 @@
.. _searx_extra:
======================================================
Tooling box ``searx_extra`` for developers and users
======================================================
In the folder :origin:`searx_extra/` we maintain some tools useful for
developers and users.
.. toctree::
:maxdepth: 2
:caption: Contents
standalone_searx.py

View File

@ -0,0 +1,9 @@
.. _standalone_searx.py:
===================================
``searx_extra/standalone_searx.py``
===================================
.. automodule:: searx_extra.standalone_searx
:members:

View File

@ -1,12 +1,11 @@
.. _searx_utils: .. _searx_utils:
.. _toolboxing: .. _toolboxing:
======================= ========================================
Tooling box ``utils/*`` Tooling box ``utils`` for administrators
======================= ========================================
In the folder :origin:`utils/` we maintain some tools useful for admins and In the folder :origin:`utils/` we maintain some tools useful for administrators.
developers.
.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
@ -16,7 +15,6 @@ developers.
filtron.sh filtron.sh
morty.sh morty.sh
lxc.sh lxc.sh
standalone_searx.py
.. _toolboxing common: .. _toolboxing common:

View File

@ -1,11 +0,0 @@
.. _standalone_searx.py:
=============================
``utils/standalone_searx.py``
=============================
.. automodule:: standalone_searx
:members:

0
searx_extra/__init__.py Normal file
View File

View File

@ -15,7 +15,7 @@ Example to use this script:
.. code:: bash .. code:: bash
$ python3 utils/standalone_searx.py rain $ python3 searx_extra/standalone_searx.py rain
Example to run it from python: Example to run it from python:

View File

View File

@ -4,11 +4,8 @@
# More info in https://ahmia.fi/blacklist/ # More info in https://ahmia.fi/blacklist/
# set path # set path
from sys import path from os.path import join
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
#
import requests import requests
from searx import searx_dir from searx import searx_dir

View File

@ -7,7 +7,6 @@ import json
# set path # set path
from sys import path from sys import path
from os.path import realpath, dirname, join from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir, settings from searx import searx_dir, settings
from searx.engines.wikidata import send_wikidata_query from searx.engines.wikidata import send_wikidata_query

View File

@ -3,13 +3,9 @@
import sys import sys
import json import json
from urllib.parse import quote, urlparse from urllib.parse import quote, urlparse
from os.path import realpath, dirname import detect_language
import cld3
from lxml.html import fromstring from lxml.html import fromstring
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx.engines.wikidata import send_wikidata_query from searx.engines.wikidata import send_wikidata_query
from searx.utils import extract_text from searx.utils import extract_text
import searx import searx

View File

@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
""" """
# pylint: disable=C0116 # pylint: disable=C0116
import sys
import json import json
import re import re
from os.path import realpath, dirname, join from os.path import join
import requests import requests
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir # pylint: disable=E0401 C0413 from searx import searx_dir # pylint: disable=E0401 C0413

View File

@ -1,14 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
#
import json import json
import requests import requests
import re import re
from os.path import dirname, join
from urllib.parse import urlparse, urljoin from urllib.parse import urlparse, urljoin
from distutils.version import LooseVersion, StrictVersion from distutils.version import LooseVersion, StrictVersion
from lxml import html from lxml import html

View File

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*- #!/usr/bin/env python
# This script generates languages.py from intersecting each engine's supported languages. # This script generates languages.py from intersecting each engine's supported languages.
# #
@ -7,11 +7,9 @@
import json import json
from pathlib import Path from pathlib import Path
from pprint import pformat from pprint import pformat
from sys import path
from babel import Locale, UnknownLocaleError from babel import Locale, UnknownLocaleError
from babel.languages import get_global from babel.languages import get_global
path.append('../searx') # noqa
from searx import settings, searx_dir from searx import settings, searx_dir
from searx.engines import initialize_engines, engines from searx.engines import initialize_engines, engines

View File

@ -4,9 +4,7 @@ import json
import collections import collections
# set path # set path
from sys import path from os.path import join
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir from searx import searx_dir
from searx.engines.wikidata import send_wikidata_query from searx.engines.wikidata import send_wikidata_query

View File

@ -41,7 +41,7 @@ setup(
author='Adam Tauber', author='Adam Tauber',
author_email='asciimoo@gmail.com', author_email='asciimoo@gmail.com',
license='GNU Affero General Public License', license='GNU Affero General Public License',
packages=find_packages(exclude=["tests*"]), packages=find_packages(exclude=["tests*", "searx_extra"]),
zip_safe=False, zip_safe=False,
install_requires=requirements, install_requires=requirements,
extras_require={ extras_require={

View File

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""Test utils/standalone_searx.py""" """Test utils/standalone_searx.py"""
import datetime import datetime
import importlib.util
import io import io
import sys import sys
@ -10,16 +9,7 @@ from nose2.tools import params
from searx.search import SearchQuery, EngineRef, initialize from searx.search import SearchQuery, EngineRef, initialize
from searx.testing import SearxTestCase from searx.testing import SearxTestCase
from searx_extra import standalone_searx as sas
def get_standalone_searx_module():
"""Get standalone_searx module."""
module_name = 'utils.standalone_searx'
filename = 'utils/standalone_searx.py'
spec = importlib.util.spec_from_file_location(module_name, filename)
sas = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sas)
return sas
class StandaloneSearx(SearxTestCase): class StandaloneSearx(SearxTestCase):
@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_no_args(self): def test_parse_argument_no_args(self):
"""Test parse argument without args.""" """Test parse argument without args."""
sas = get_standalone_searx_module()
with patch.object(sys, 'argv', ['standalone_searx']), \ with patch.object(sys, 'argv', ['standalone_searx']), \
self.assertRaises(SystemExit): self.assertRaises(SystemExit):
sys.stderr = io.StringIO() sys.stderr = io.StringIO()
@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_basic_args(self): def test_parse_argument_basic_args(self):
"""Test parse argument with basic args.""" """Test parse argument with basic args."""
sas = get_standalone_searx_module()
query = 'red box' query = 'red box'
exp_dict = { exp_dict = {
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1, 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict(self): def test_to_dict(self):
"""test to_dict.""" """test to_dict."""
sas = get_standalone_searx_module()
self.assertEqual( self.assertEqual(
sas.to_dict( sas.to_dict(
sas.get_search_query(sas.parse_argument(['red box']))), sas.get_search_query(sas.parse_argument(['red box']))),
@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict_with_mock(self): def test_to_dict_with_mock(self):
"""test to dict.""" """test to dict."""
sas = get_standalone_searx_module()
with patch.object(sas.searx.search, 'Search') as mock_s: with patch.object(sas.searx.search, 'Search') as mock_s:
m_search = mock_s().search() m_search = mock_s().search()
m_sq = Mock() m_sq = Mock()
@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
def test_get_search_query(self): def test_get_search_query(self):
"""test get_search_query.""" """test get_search_query."""
sas = get_standalone_searx_module()
args = sas.parse_argument(['rain', ]) args = sas.parse_argument(['rain', ])
search_q = sas.get_search_query(args) search_q = sas.get_search_query(args)
self.assertTrue(search_q) self.assertTrue(search_q)
@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
def test_no_parsed_url(self): def test_no_parsed_url(self):
"""test no_parsed_url func""" """test no_parsed_url func"""
sas = get_standalone_searx_module()
self.assertEqual( self.assertEqual(
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]), sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
[{}] [{}]
@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
) )
def test_json_serial(self, arg, exp_res): def test_json_serial(self, arg, exp_res):
"""test json_serial func""" """test json_serial func"""
sas = get_standalone_searx_module()
self.assertEqual(sas.json_serial(arg), exp_res) self.assertEqual(sas.json_serial(arg), exp_res)
def test_json_serial_error(self): def test_json_serial_error(self):
"""test error on json_serial.""" """test error on json_serial."""
sas = get_standalone_searx_module()
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
sas.json_serial('a') sas.json_serial('a')