Merge branch 'master' into multiple-architecture-support

.github/workflows/data-update.yml

@@ -0,0 +1,67 @@
+name: "Update searx.data"
+on:
+  schedule:
+    - cron: "37 13 * * 0"
+
+jobs:
+  updateData:
+    name: Update data
+    runs-on: ubuntu-20.04
+    if: env.DATA_PR_TOKEN != null
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Ubuntu packages
+        run: |
+          sudo ./utils/searx.sh install packages
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+
+      - name: Cache Python dependencies
+        id: cache-python
+        uses: actions/cache@v2
+        with:
+          path: ./local
+          key: python-${{ matrix.os }}-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    
+      - name: Install Python dependencies
+        if: steps.cache-python.outputs.cache-hit != 'true'
+        run: |
+          make V=1 install
+
+      - name: Fetch data
+        run: |
+          source local/py3/bin/activate
+          python utils/fetch_firefox_version.py
+          python utils/fetch_languages.py
+          python utils/fetch_ahmia_blacklist.py
+          python utils/fetch_wikidata_units.py
+          # python utils/fetch_currencies.py
+
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.DATA_PR_TOKEN }}
+          commit-message: Update searx.data
+          committer: searx-bot <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: automatic-update-data
+          delete-branch: true
+          title: 'Update searx.data'
+          body: |
+            Update searx.data
+          labels: |
+            data
+          draft: false
+
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.github/workflows/integration.yml

@@ -34,7 +34,7 @@ jobs:
         make V=1 install
         make V=1 gecko.driver
     - name: Run tests
-      run: make V=1 test
+      run: make V=1 ci.test
     - name: Test coverage
       run: make V=1 test.coverage
     - name: Store coverage result
@@ -80,7 +80,7 @@ jobs:
       with:
         GITHUB_TOKEN: ${{ github.token }}
         BRANCH: gh-pages
-        FOLDER: gh-pages
+        FOLDER: build/gh-pages
         CLEAN: true # Automatically remove deleted files from the deploy branch
 
   dockers:

Makefile

@@ -251,6 +251,11 @@ test.clean:
 # travis
 # ------
 
+PHONY += ci.test
+ci.test:
+	$(PY_ENV_BIN)/python -c "import yaml"  || make clean
+	$(MAKE) test
+
 travis.codecov:
 	$(Q)$(PY_ENV_BIN)/python -m pip install codecov
 

docs/dev/engine_overview.rst

@@ -42,7 +42,6 @@ argument                type        information
 ======================= =========== ========================================================
 categories              list        pages, in which the engine is working
 paging                  boolean     support multible pages
-language_support        boolean     support language choosing
 time_range_support      boolean     support search time range
 engine_type             str         ``online`` by default, other possibles values are 
                                     ``offline``, ``online_dictionnary``, ``online_currency``
@@ -97,7 +96,6 @@ example code
    # engine dependent config
    categories = ['general']
    paging = True
-   language_support = True
 
 
 making a request

searx/engines/__init__.py

@@ -44,7 +44,6 @@ babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else
 engine_shortcuts = {}
 engine_default_args = {'paging': False,
                        'categories': ['general'],
-                       'language_support': True,
                        'supported_languages': [],
                        'safesearch': False,
                        'timeout': settings['outgoing']['request_timeout'],
@@ -127,6 +126,9 @@ def load_engine(engine_data):
 
         setattr(engine, 'language_aliases', language_aliases)
 
+    # language_support
+    setattr(engine, 'language_support', len(getattr(engine, 'supported_languages', [])) > 0)
+
     # assign language fetching method if auxiliary method exists
     if hasattr(engine, '_fetch_supported_languages'):
         setattr(engine, 'fetch_supported_languages',

searx/engines/archlinux.py

@@ -21,7 +21,6 @@ about = {
 
 # engine dependent config
 categories = ['it']
-language_support = True
 paging = True
 base_url = 'https://wiki.archlinux.org'
 

searx/engines/bing.py

@@ -24,7 +24,6 @@ about = {
 # engine dependent config
 categories = ['general']
 paging = True
-language_support = True
 supported_languages_url = 'https://www.bing.com/account/general'
 language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
 

searx/engines/bing_images.py

@@ -26,7 +26,6 @@ categories = ['images']
 paging = True
 safesearch = True
 time_range_support = True
-language_support = True
 supported_languages_url = 'https://www.bing.com/account/general'
 number_of_results = 28
 

searx/engines/bing_news.py

@@ -25,7 +25,6 @@ about = {
 # engine dependent config
 categories = ['news']
 paging = True
-language_support = True
 time_range_support = True
 
 # search-url

searx/engines/bing_videos.py

@@ -26,7 +26,6 @@ paging = True
 safesearch = True
 time_range_support = True
 number_of_results = 28
-language_support = True
 
 base_url = 'https://www.bing.com/'
 search_string = 'videos/search'\

searx/engines/dailymotion.py

@@ -21,7 +21,6 @@ about = {
 # engine dependent config
 categories = ['videos']
 paging = True
-language_support = True
 
 # search-url
 # see http://www.dailymotion.com/doc/api/obj-video.html

searx/engines/doku.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['general']  # TODO , 'images', 'music', 'videos', 'files'
 paging = False
-language_support = False
 number_of_results = 5
 
 # search-url

searx/engines/duckduckgo.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['general']
 paging = False
-language_support = True
 supported_languages_url = 'https://duckduckgo.com/util/u172.js'
 time_range_support = True
 

searx/engines/duckduckgo_images.py

@@ -26,7 +26,6 @@ about = {
 # engine dependent config
 categories = ['images']
 paging = True
-language_support = True
 safesearch = True
 
 # search-url

searx/engines/duden.py

@@ -20,7 +20,6 @@ about = {
 
 categories = ['general']
 paging = True
-language_support = False
 
 # search-url
 base_url = 'https://www.duden.de/'

searx/engines/etools.py

@@ -19,7 +19,6 @@ about = {
 
 categories = ['general']
 paging = False
-language_support = False
 safesearch = True
 
 base_url = 'https://www.etools.ch'

searx/engines/genius.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['music']
 paging = True
-language_support = False
 page_size = 5
 
 url = 'https://genius.com/api/'

searx/engines/gentoo.py

@@ -19,7 +19,6 @@ about = {
 
 # engine dependent config
 categories = ['it']
-language_support = True
 paging = True
 base_url = 'https://wiki.gentoo.org'
 

searx/engines/gigablast.py

@@ -24,7 +24,6 @@ about = {
 categories = ['general']
 # gigablast's pagination is totally damaged, don't use it
 paging = False
-language_support = True
 safesearch = True
 
 # search-url

searx/engines/google.py

@@ -31,7 +31,6 @@ about = {
 # engine dependent config
 categories = ['general']
 paging = True
-language_support = True
 time_range_support = True
 safesearch = True
 supported_languages_url = 'https://www.google.com/preferences?#languages'

searx/engines/google_images.py

@@ -56,7 +56,6 @@ about = {
 # engine dependent config
 categories = ['images']
 paging = False
-language_support = True
 use_locale_domain = True
 time_range_support = True
 safesearch = True

searx/engines/google_news.py

@@ -68,7 +68,6 @@ time_range_dict = {
 
 categories = ['news']
 paging = False
-language_support = True
 use_locale_domain = True
 time_range_support = True
 

searx/engines/invidious.py

@@ -21,7 +21,6 @@ about = {
 # engine dependent config
 categories = ["videos", "music"]
 paging = True
-language_support = True
 time_range_support = True
 
 

searx/engines/mediawiki.py

@@ -19,7 +19,6 @@ about = {
 
 # engine dependent config
 categories = ['general']
-language_support = True
 paging = True
 number_of_results = 1
 search_type = 'nearmatch'  # possible values: title, text, nearmatch

searx/engines/peertube.py

@@ -21,7 +21,6 @@ about = {
 # engine dependent config
 categories = ["videos"]
 paging = True
-language_support = True
 base_url = "https://peer.tube/"
 supported_languages_url = base_url + "api/v1/videos/languages"
 

searx/engines/photon.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['map']
 paging = False
-language_support = True
 number_of_results = 10
 
 # search-url

searx/engines/qwant.py

@@ -23,7 +23,6 @@ about = {
 # engine dependent config
 categories = []
 paging = True
-language_support = True
 supported_languages_url = 'https://qwant.com/region'
 
 category_to_keyword = {'general': 'web',

searx/engines/sepiasearch.py

@@ -20,7 +20,6 @@ about = {
 
 categories = ['videos']
 paging = True
-language_support = True
 time_range_support = True
 safesearch = True
 supported_languages = [

searx/engines/startpage.py

@@ -29,7 +29,6 @@ categories = ['general']
 # storing of qid's between mulitble search-calls
 
 paging = True
-language_support = True
 supported_languages_url = 'https://www.startpage.com/do/settings'
 
 # search-url

searx/engines/yacy.py

@@ -24,7 +24,6 @@ about = {
 # engine dependent config
 categories = ['general', 'images']  # TODO , 'music', 'videos', 'files'
 paging = True
-language_support = True
 number_of_results = 5
 http_digest_auth_user = ""
 http_digest_auth_pass = ""

searx/engines/yahoo.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['general']
 paging = True
-language_support = True
 time_range_support = True
 
 # search-url

searx/engines/yahoo_news.py

@@ -25,7 +25,6 @@ about = {
 # engine dependent config
 categories = ['news']
 paging = True
-language_support = True
 
 # search-url
 search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&{lang}=uh3_news_web_gs_1&pz=10&xargs=0&vl=lang_{lang}'  # noqa

searx/engines/yandex.py

@@ -23,7 +23,6 @@ about = {
 # engine dependent config
 categories = ['general']
 paging = True
-language_support = True  # TODO
 
 default_tld = 'com'
 language_map = {'ru': 'ru',

searx/engines/youtube_api.py

@@ -21,7 +21,6 @@ about = {
 # engine dependent config
 categories = ['videos', 'music']
 paging = False
-language_support = True
 api_key = None
 
 # search-url

searx/engines/youtube_noapi.py

@@ -20,7 +20,6 @@ about = {
 # engine dependent config
 categories = ['videos', 'music']
 paging = True
-language_support = False
 time_range_support = True
 
 # search-url

utils/fabfile.py

@@ -1,117 +0,0 @@
-from fabric.api import cd, run, sudo, put
-from io import StringIO
-
-
-base_dir = '/usr/local'
-hostname = 'searx.me'
-searx_dir = base_dir + '/searx'
-searx_ve_dir = searx_dir + '/searx-ve'
-current_user = run('whoami').stdout.strip()
-
-uwsgi_file = '''
-[uwsgi]
-# Who will run the code
-uid = {user}
-gid = {user}
-
-# Number of workers
-workers = 8
-
-# The right granted on the created socket
-chmod-socket = 666
-
-# Plugin to use and interpretor config
-single-interpreter = true
-master = true
-plugin = python
-
-# Module to import
-module = searx.webapp
-
-# Virtualenv and python path
-virtualenv = {searx_ve_dir}
-pythonpath = {searx_dir}
-chdir = {searx_dir}/searx
-'''.format(user=current_user,
-           searx_dir=searx_dir,
-           searx_ve_dir=searx_ve_dir)
-
-nginx_config = '''
-server {{
-    listen 80;
-    server_name {hostname};
-    server_name www.{hostname};
-    root /usr/local/searx;
-
-    location / {{
-        include uwsgi_params;
-        uwsgi_pass unix:/run/uwsgi/app/searx/socket;
-    }}
-}}
-'''.format(hostname=hostname)
-
-
-def stop():
-    sudo('/etc/init.d/uwsgi stop')
-
-
-def start():
-    sudo('/etc/init.d/uwsgi start')
-
-
-def restart():
-    sudo('/etc/init.d/uwsgi restart')
-
-
-def init():
-    if not run('test -d ' + searx_dir, warn_only=True).failed:
-        return
-
-    sudo('apt-get update')
-
-    sudo('apt-get install git'
-         ' build-essential'
-         ' libxslt-dev'
-         ' python-dev'
-         ' python-virtualenv'
-         ' python-pybabel'
-         ' zlib1g-dev'
-         ' uwsgi'
-         ' uwsgi-plugin-python'
-         ' nginx')
-
-    sudo('mkdir -p ' + base_dir)
-
-    put(StringIO(nginx_config), '/etc/nginx/sites-enabled/searx', use_sudo=True)
-    sudo('/etc/init.d/nginx restart')
-
-    with cd(base_dir):
-        sudo('git clone https://github.com/searx/searx')
-
-    sudo('chown -R {user}:{user} {searx_dir}'.format(user=current_user, searx_dir=searx_dir))
-    put(StringIO(uwsgi_file), searx_dir + '/uwsgi.ini')
-    sudo('ln -s {0}/uwsgi.ini /etc/uwsgi/apps-enabled/searx.ini'.format(searx_dir))
-
-    run('virtualenv {0}'.format(searx_ve_dir))
-
-    with cd(searx_dir):
-        run('source {0}/bin/activate && pip install -r requirements.txt'.format(searx_ve_dir))
-
-    start()
-
-
-def deploy():
-    init()
-
-    with cd(searx_dir):
-        run("git stash", warn_only=True)
-        run("git pull origin master")
-        run("git stash pop", warn_only=True)
-
-    restart()
-
-
-def clean():
-    sudo('rm -rf {searx_dir}'.format(searx_dir=searx_dir), warn_only=True)
-    sudo('rm /etc/uwsgi/apps-enabled/searx.ini', warn_only=True)
-    sudo('rm /etc/nginx/sites-enabled/searx', warn_only=True)

utils/fetch_languages.py

@@ -6,18 +6,19 @@
 # are written in current directory to avoid overwriting in case something goes wrong.
 
 import json
+from pathlib import Path
 from pprint import pformat
 from sys import path
 from babel import Locale, UnknownLocaleError
 from babel.languages import get_global
 
 path.append('../searx')  # noqa
-from searx import settings
+from searx import settings, searx_dir
 from searx.engines import initialize_engines, engines
 
 # Output files.
-engines_languages_file = 'engines_languages.json'
+engines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json'
-languages_file = 'languages.py'
+languages_file = Path(searx_dir) / 'languages.py'
 
 
 # Fetchs supported languages for each engine and writes json file with those.

utils/makefile.python

@@ -87,6 +87,22 @@ python-exe:
 	@:
 endif
 
+msg-pip-exe:
+	@echo "\n  $(PIP) is required\n\n\
+  Make sure you have updated pip installed, grab it from\n\
+  https://pip.pypa.io or install it from your package\n\
+  manager. On debian based OS these requirements are\n\
+  installed by::\n\n\
+    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)
+
+ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
+pip-exe: msg-pip-exe
+	$(error The '$(PIP)' command was not found)
+else
+pip-exe:
+	@:
+endif
+
 # ------------------------------------------------------------------------------
 # commands
 # ------------------------------------------------------------------------------
@@ -99,7 +115,8 @@ quiet_cmd_pyinstall = INSTALL   $2
 quiet_cmd_pyenvinstall = PYENV     install $2
       cmd_pyenvinstall = \
 	if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \
-		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) ;\
+		rm -f $(PY_ENV)/requirements.sha256; \
+		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\
 		sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\
 	else \
 		echo "PYENV     $2 already installed"; \
@@ -119,13 +136,13 @@ quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
 # $2 path to folder where virtualenv take place
 quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
       cmd_virtualenv  = \
-	if [ ! -d "./$(PY_ENV)" ];then                                  \
+	if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \
-		$(PYTHON) -m venv $(VTENV_OPTS) $2;                     \
+		echo "PYENV     using virtualenv from $2"; \
+	else \
+		$(PYTHON) -m venv $(VTENV_OPTS) $2; \
 		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \
-		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt;     \
+		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \
-	else                                                            \
+	fi
-		echo "PYENV     using virtualenv from $2";              \
-        fi
 
 # $2 path to lint
 quiet_cmd_pylint      = LINT      $@

utils/makefile.sphinx

@@ -10,7 +10,7 @@ SPHINX_CONF ?= conf.py
 DOCS_FOLDER = ./docs
 DOCS_BUILD  = ./$(LXC_ENV_FOLDER)build/docs
 DOCS_DIST   = ./$(LXC_ENV_FOLDER)dist/docs
-GH_PAGES    ?= gh-pages
+GH_PAGES    ?= build/gh-pages
 
 BOOKS_FOLDER = ./docs
 BOOKS_DIST   = ./$(LXC_ENV_FOLDER)dist/books
@@ -172,24 +172,22 @@ PHONY += prepare-gh-pages
 prepare-gh-pages:
 	cp -r $(DOCS_DIST)/* $(GH_PAGES)/
 	touch $(GH_PAGES)/.nojekyll
-	echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html	
+	echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html
 
-PHONY += $(GH_PAGES)
+PHONY += gh-pages
-$(GH_PAGES)::
+gh-pages: docs-clean docs
-	$(MAKE) docs
+	- git worktree remove -f $(GH_PAGES) || exit 0
-	[ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages
+	- git branch -D gh-pages || exit 0
-	-cd $(GH_PAGES); git checkout gh-pages >/dev/null
+	git worktree add --no-checkout $(GH_PAGES) master
-	-cd $(GH_PAGES); git pull
+	cd $(GH_PAGES); git checkout --orphan gh-pages && git rm -rfq .
-	-cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf
 	$(MAKE) prepare-gh-pages
 	cd $(GH_PAGES);\
 		git add --all . ;\
-		git commit -m "gh-pages: updated" ;\
+		git commit -q -m "make gh-pages: from $(shell git config --get remote.origin.url)@$(shell git rev-parse HEAD)" ;\
-		git push origin gh-pages
+		git push -f origin gh-pages
 
 PHONY += travis-gh-pages
-travis-gh-pages:
+travis-gh-pages: docs-clean docs
-	$(MAKE) docs
 	rm -Rf $(GH_PAGES)
 	mkdir -p $(GH_PAGES)
 	$(MAKE) prepare-gh-pages