mirror of https://github.com/searx/searx
Add searx_extra package
Split the utils directory into: * searx_extra contains update scripts, standalone_searx.py * utils contains the files to build and setup searx.
This commit is contained in:
parent
111d38cd8f
commit
b8cd326464
|
@ -11,12 +11,12 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
fetch:
|
||||
- ahmia_blacklist
|
||||
- currencies
|
||||
- external_bangs
|
||||
- firefox_version
|
||||
- languages
|
||||
- wikidata_units
|
||||
- update_ahmia_blacklist.py
|
||||
- update_currencies.py
|
||||
- update_external_bangs.py
|
||||
- update_firefox_version.py
|
||||
- update_languages.py
|
||||
- update_wikidata_units.py
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
@ -45,10 +45,10 @@ jobs:
|
|||
|
||||
- name: Fetch data
|
||||
env:
|
||||
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
|
||||
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
|
||||
run: |
|
||||
source local/py3/bin/activate
|
||||
python $FETCH_SCRIPT
|
||||
$FETCH_SCRIPT
|
||||
|
||||
- name: Create Pull Request
|
||||
id: cpr
|
||||
|
|
4
Makefile
4
Makefile
|
@ -195,8 +195,8 @@ PYLINT_FILES=\
|
|||
searx/engines/google_videos.py \
|
||||
searx/engines/google_images.py \
|
||||
searx/engines/mediathekviewweb.py \
|
||||
utils/fetch_external_bangs.py \
|
||||
searx/engines/google_scholar.py
|
||||
searx/engines/google_scholar.py \
|
||||
searx_extra/update/update_external_bangs.py
|
||||
|
||||
test.pylint: pyenvinstall
|
||||
$(call cmd,pylint,$(PYLINT_FILES))
|
||||
|
|
|
@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
|
|||
user/index
|
||||
admin/index
|
||||
dev/index
|
||||
searx_extra/index
|
||||
utils/index
|
||||
blog/index
|
||||
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
.. _searx_extra:
|
||||
|
||||
======================================================
|
||||
Tooling box ``searx_extra`` for developers and users
|
||||
======================================================
|
||||
|
||||
In the folder :origin:`searx_extra/` we maintain some tools useful for
|
||||
developers and users.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
standalone_searx.py
|
|
@ -0,0 +1,9 @@
|
|||
|
||||
.. _standalone_searx.py:
|
||||
|
||||
===================================
|
||||
``searx_extra/standalone_searx.py``
|
||||
===================================
|
||||
|
||||
.. automodule:: searx_extra.standalone_searx
|
||||
:members:
|
|
@ -1,12 +1,11 @@
|
|||
.. _searx_utils:
|
||||
.. _toolboxing:
|
||||
|
||||
=======================
|
||||
Tooling box ``utils/*``
|
||||
=======================
|
||||
========================================
|
||||
Tooling box ``utils`` for administrators
|
||||
========================================
|
||||
|
||||
In the folder :origin:`utils/` we maintain some tools useful for admins and
|
||||
developers.
|
||||
In the folder :origin:`utils/` we maintain some tools useful for administrators.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
@ -16,7 +15,6 @@ developers.
|
|||
filtron.sh
|
||||
morty.sh
|
||||
lxc.sh
|
||||
standalone_searx.py
|
||||
|
||||
.. _toolboxing common:
|
||||
|
||||
|
|
|
@ -1,11 +0,0 @@
|
|||
|
||||
.. _standalone_searx.py:
|
||||
|
||||
=============================
|
||||
``utils/standalone_searx.py``
|
||||
=============================
|
||||
|
||||
.. automodule:: standalone_searx
|
||||
:members:
|
||||
|
||||
|
|
@ -15,7 +15,7 @@ Example to use this script:
|
|||
|
||||
.. code:: bash
|
||||
|
||||
$ python3 utils/standalone_searx.py rain
|
||||
$ python3 searx_extra/standalone_searx.py rain
|
||||
|
||||
Example to run it from python:
|
||||
|
|
@ -4,11 +4,8 @@
|
|||
# More info in https://ahmia.fi/blacklist/
|
||||
|
||||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname, join
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
from os.path import join
|
||||
|
||||
#
|
||||
import requests
|
||||
from searx import searx_dir
|
||||
|
|
@ -7,7 +7,6 @@ import json
|
|||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname, join
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
from searx import searx_dir, settings
|
||||
from searx.engines.wikidata import send_wikidata_query
|
6
utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py
Normal file → Executable file
6
utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py
Normal file → Executable file
|
@ -3,13 +3,9 @@
|
|||
import sys
|
||||
import json
|
||||
from urllib.parse import quote, urlparse
|
||||
from os.path import realpath, dirname
|
||||
import cld3
|
||||
import detect_language
|
||||
from lxml.html import fromstring
|
||||
|
||||
# set path
|
||||
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
from searx.engines.wikidata import send_wikidata_query
|
||||
from searx.utils import extract_text
|
||||
import searx
|
|
@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
|
|||
"""
|
||||
# pylint: disable=C0116
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
from os.path import realpath, dirname, join
|
||||
from os.path import join
|
||||
|
||||
import requests
|
||||
|
||||
# set path
|
||||
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
from searx import searx_dir # pylint: disable=E0401 C0413
|
||||
|
||||
|
|
@ -1,14 +1,9 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname, join
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
#
|
||||
import json
|
||||
import requests
|
||||
import re
|
||||
from os.path import dirname, join
|
||||
from urllib.parse import urlparse, urljoin
|
||||
from distutils.version import LooseVersion, StrictVersion
|
||||
from lxml import html
|
|
@ -1,4 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#!/usr/bin/env python
|
||||
|
||||
# This script generates languages.py from intersecting each engine's supported languages.
|
||||
#
|
||||
|
@ -7,11 +7,9 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
from sys import path
|
||||
from babel import Locale, UnknownLocaleError
|
||||
from babel.languages import get_global
|
||||
|
||||
path.append('../searx') # noqa
|
||||
from searx import settings, searx_dir
|
||||
from searx.engines import initialize_engines, engines
|
||||
|
4
utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py
Normal file → Executable file
4
utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py
Normal file → Executable file
|
@ -4,9 +4,7 @@ import json
|
|||
import collections
|
||||
|
||||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname, join
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
from os.path import join
|
||||
|
||||
from searx import searx_dir
|
||||
from searx.engines.wikidata import send_wikidata_query
|
2
setup.py
2
setup.py
|
@ -41,7 +41,7 @@ setup(
|
|||
author='Adam Tauber',
|
||||
author_email='asciimoo@gmail.com',
|
||||
license='GNU Affero General Public License',
|
||||
packages=find_packages(exclude=["tests*"]),
|
||||
packages=find_packages(exclude=["tests*", "searx_extra"]),
|
||||
zip_safe=False,
|
||||
install_requires=requirements,
|
||||
extras_require={
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Test utils/standalone_searx.py"""
|
||||
import datetime
|
||||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
|
||||
|
@ -10,16 +9,7 @@ from nose2.tools import params
|
|||
|
||||
from searx.search import SearchQuery, EngineRef, initialize
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
def get_standalone_searx_module():
|
||||
"""Get standalone_searx module."""
|
||||
module_name = 'utils.standalone_searx'
|
||||
filename = 'utils/standalone_searx.py'
|
||||
spec = importlib.util.spec_from_file_location(module_name, filename)
|
||||
sas = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(sas)
|
||||
return sas
|
||||
from searx_extra import standalone_searx as sas
|
||||
|
||||
|
||||
class StandaloneSearx(SearxTestCase):
|
||||
|
@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_parse_argument_no_args(self):
|
||||
"""Test parse argument without args."""
|
||||
sas = get_standalone_searx_module()
|
||||
with patch.object(sys, 'argv', ['standalone_searx']), \
|
||||
self.assertRaises(SystemExit):
|
||||
sys.stderr = io.StringIO()
|
||||
|
@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_parse_argument_basic_args(self):
|
||||
"""Test parse argument with basic args."""
|
||||
sas = get_standalone_searx_module()
|
||||
query = 'red box'
|
||||
exp_dict = {
|
||||
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
|
||||
|
@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_to_dict(self):
|
||||
"""test to_dict."""
|
||||
sas = get_standalone_searx_module()
|
||||
self.assertEqual(
|
||||
sas.to_dict(
|
||||
sas.get_search_query(sas.parse_argument(['red box']))),
|
||||
|
@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_to_dict_with_mock(self):
|
||||
"""test to dict."""
|
||||
sas = get_standalone_searx_module()
|
||||
with patch.object(sas.searx.search, 'Search') as mock_s:
|
||||
m_search = mock_s().search()
|
||||
m_sq = Mock()
|
||||
|
@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_get_search_query(self):
|
||||
"""test get_search_query."""
|
||||
sas = get_standalone_searx_module()
|
||||
args = sas.parse_argument(['rain', ])
|
||||
search_q = sas.get_search_query(args)
|
||||
self.assertTrue(search_q)
|
||||
|
@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
|
|||
|
||||
def test_no_parsed_url(self):
|
||||
"""test no_parsed_url func"""
|
||||
sas = get_standalone_searx_module()
|
||||
self.assertEqual(
|
||||
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
|
||||
[{}]
|
||||
|
@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
|
|||
)
|
||||
def test_json_serial(self, arg, exp_res):
|
||||
"""test json_serial func"""
|
||||
sas = get_standalone_searx_module()
|
||||
self.assertEqual(sas.json_serial(arg), exp_res)
|
||||
|
||||
def test_json_serial_error(self):
|
||||
"""test error on json_serial."""
|
||||
sas = get_standalone_searx_module()
|
||||
with self.assertRaises(TypeError):
|
||||
sas.json_serial('a')
|
||||
|
|
Loading…
Reference in New Issue