Browse Source

Add searx_extra package

Split the utils directory into:
* searx_extra contains update scripts, standalone_searx.py
* utils contains the files to build and setup searx.
pull/2600/head
Alexandre Flament 8 months ago
parent
commit
b8cd326464
  1. 16
      .github/workflows/data-update.yml
  2. 4
      Makefile
  3. 1
      docs/index.rst
  4. 14
      docs/searx_extra/index.rst
  5. 9
      docs/searx_extra/standalone_searx.py.rst
  6. 10
      docs/utils/index.rst
  7. 11
      docs/utils/standalone_searx.py.rst
  8. 0
      searx_extra/__init__.py
  9. 0
      searx_extra/google_search.py
  10. 2
      searx_extra/standalone_searx.py
  11. 0
      searx_extra/update/__init__.py
  12. 5
      searx_extra/update/update_ahmia_blacklist.py
  13. 1
      searx_extra/update/update_currencies.py
  14. 6
      searx_extra/update/update_engine_descriptions.py
  15. 6
      searx_extra/update/update_external_bangs.py
  16. 7
      searx_extra/update/update_firefox_version.py
  17. 4
      searx_extra/update/update_languages.py
  18. 0
      searx_extra/update/update_translations.sh
  19. 4
      searx_extra/update/update_wikidata_units.py
  20. 2
      setup.py
  21. 20
      tests/unit/test_standalone_searx.py

16
.github/workflows/data-update.yml

@ -11,12 +11,12 @@ jobs:
strategy:
matrix:
fetch:
- ahmia_blacklist
- currencies
- external_bangs
- firefox_version
- languages
- wikidata_units
- update_ahmia_blacklist.py
- update_currencies.py
- update_external_bangs.py
- update_firefox_version.py
- update_languages.py
- update_wikidata_units.py
steps:
- name: Checkout
uses: actions/checkout@v2
@ -45,10 +45,10 @@ jobs:
- name: Fetch data
env:
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
run: |
source local/py3/bin/activate
python $FETCH_SCRIPT
$FETCH_SCRIPT
- name: Create Pull Request
id: cpr

4
Makefile

@ -195,8 +195,8 @@ PYLINT_FILES=\
searx/engines/google_videos.py \
searx/engines/google_images.py \
searx/engines/mediathekviewweb.py \
utils/fetch_external_bangs.py \
searx/engines/google_scholar.py
searx/engines/google_scholar.py \
searx_extra/update/update_external_bangs.py
test.pylint: pyenvinstall
$(call cmd,pylint,$(PYLINT_FILES))

1
docs/index.rst

@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
user/index
admin/index
dev/index
searx_extra/index
utils/index
blog/index

14
docs/searx_extra/index.rst

@ -0,0 +1,14 @@
.. _searx_extra:
======================================================
Tooling box ``searx_extra`` for developers and users
======================================================
In the folder :origin:`searx_extra/` we maintain some tools useful for
developers and users.
.. toctree::
:maxdepth: 2
:caption: Contents
standalone_searx.py

9
docs/searx_extra/standalone_searx.py.rst

@ -0,0 +1,9 @@
.. _standalone_searx.py:
===================================
``searx_extra/standalone_searx.py``
===================================
.. automodule:: searx_extra.standalone_searx
:members:

10
docs/utils/index.rst

@ -1,12 +1,11 @@
.. _searx_utils:
.. _toolboxing:
=======================
Tooling box ``utils/*``
=======================
========================================
Tooling box ``utils`` for administrators
========================================
In the folder :origin:`utils/` we maintain some tools useful for admins and
developers.
In the folder :origin:`utils/` we maintain some tools useful for administrators.
.. toctree::
:maxdepth: 2
@ -16,7 +15,6 @@ developers.
filtron.sh
morty.sh
lxc.sh
standalone_searx.py
.. _toolboxing common:

11
docs/utils/standalone_searx.py.rst

@ -1,11 +0,0 @@
.. _standalone_searx.py:
=============================
``utils/standalone_searx.py``
=============================
.. automodule:: standalone_searx
:members:

0
searx_extra/__init__.py

0
utils/google_search.py → searx_extra/google_search.py

2
utils/standalone_searx.py → searx_extra/standalone_searx.py

@ -15,7 +15,7 @@ Example to use this script:
.. code:: bash
$ python3 utils/standalone_searx.py rain
$ python3 searx_extra/standalone_searx.py rain
Example to run it from python:

0
searx_extra/update/__init__.py

5
utils/fetch_ahmia_blacklist.py → searx_extra/update/update_ahmia_blacklist.py

@ -4,11 +4,8 @@
# More info in https://ahmia.fi/blacklist/
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join
#
import requests
from searx import searx_dir

1
utils/fetch_currencies.py → searx_extra/update/update_currencies.py

@ -7,7 +7,6 @@ import json
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir, settings
from searx.engines.wikidata import send_wikidata_query

6
utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py

@ -3,13 +3,9 @@
import sys
import json
from urllib.parse import quote, urlparse
from os.path import realpath, dirname
import cld3
import detect_language
from lxml.html import fromstring
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx.engines.wikidata import send_wikidata_query
from searx.utils import extract_text
import searx

6
utils/fetch_external_bangs.py → searx_extra/update/update_external_bangs.py

@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
"""
# pylint: disable=C0116
import sys
import json
import re
from os.path import realpath, dirname, join
from os.path import join
import requests
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir # pylint: disable=E0401 C0413

7
utils/fetch_firefox_version.py → searx_extra/update/update_firefox_version.py

@ -1,14 +1,9 @@
#!/usr/bin/env python
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
#
import json
import requests
import re
from os.path import dirname, join
from urllib.parse import urlparse, urljoin
from distutils.version import LooseVersion, StrictVersion
from lxml import html

4
utils/fetch_languages.py → searx_extra/update/update_languages.py

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# This script generates languages.py from intersecting each engine's supported languages.
#
@ -7,11 +7,9 @@
import json
from pathlib import Path
from pprint import pformat
from sys import path
from babel import Locale, UnknownLocaleError
from babel.languages import get_global
path.append('../searx') # noqa
from searx import settings, searx_dir
from searx.engines import initialize_engines, engines

0
utils/update-translations.sh → searx_extra/update/update_translations.sh

4
utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py

@ -4,9 +4,7 @@ import json
import collections
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join
from searx import searx_dir
from searx.engines.wikidata import send_wikidata_query

2
setup.py

@ -41,7 +41,7 @@ setup(
author='Adam Tauber',
author_email='asciimoo@gmail.com',
license='GNU Affero General Public License',
packages=find_packages(exclude=["tests*"]),
packages=find_packages(exclude=["tests*", "searx_extra"]),
zip_safe=False,
install_requires=requirements,
extras_require={

20
tests/unit/test_standalone_searx.py

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Test utils/standalone_searx.py"""
import datetime
import importlib.util
import io
import sys
@ -10,16 +9,7 @@ from nose2.tools import params
from searx.search import SearchQuery, EngineRef, initialize
from searx.testing import SearxTestCase
def get_standalone_searx_module():
"""Get standalone_searx module."""
module_name = 'utils.standalone_searx'
filename = 'utils/standalone_searx.py'
spec = importlib.util.spec_from_file_location(module_name, filename)
sas = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sas)
return sas
from searx_extra import standalone_searx as sas
class StandaloneSearx(SearxTestCase):
@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_no_args(self):
"""Test parse argument without args."""
sas = get_standalone_searx_module()
with patch.object(sys, 'argv', ['standalone_searx']), \
self.assertRaises(SystemExit):
sys.stderr = io.StringIO()
@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_basic_args(self):
"""Test parse argument with basic args."""
sas = get_standalone_searx_module()
query = 'red box'
exp_dict = {
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict(self):
"""test to_dict."""
sas = get_standalone_searx_module()
self.assertEqual(
sas.to_dict(
sas.get_search_query(sas.parse_argument(['red box']))),
@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict_with_mock(self):
"""test to dict."""
sas = get_standalone_searx_module()
with patch.object(sas.searx.search, 'Search') as mock_s:
m_search = mock_s().search()
m_sq = Mock()
@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
def test_get_search_query(self):
"""test get_search_query."""
sas = get_standalone_searx_module()
args = sas.parse_argument(['rain', ])
search_q = sas.get_search_query(args)
self.assertTrue(search_q)
@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
def test_no_parsed_url(self):
"""test no_parsed_url func"""
sas = get_standalone_searx_module()
self.assertEqual(
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
[{}]
@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
)
def test_json_serial(self, arg, exp_res):
"""test json_serial func"""
sas = get_standalone_searx_module()
self.assertEqual(sas.json_serial(arg), exp_res)
def test_json_serial_error(self):
"""test error on json_serial."""
sas = get_standalone_searx_module()
with self.assertRaises(TypeError):
sas.json_serial('a')

Loading…
Cancel
Save