Merge branch 'searx:master' into master

README.rst

@@ -85,8 +85,8 @@ of turning it off.
 What is the difference between searx and SearxNG?
 #################################################
 
-TL;DR: If you want to run a public instance, go with SearxNG. If you want to
-self host your own instance, choose searx.
+TL;DR: SearXNG is for users that want more features and bugs getting fixed quicker.
+If you prefer a minimalist software and stable experience, use searx.
 
 SearxNG is a fork of searx, created by a former maintainer of searx. The fork
 was created because the majority of the maintainers at the time did not find

manage

@@ -284,9 +284,6 @@ node.env() {
     which npm &> /dev/null || die 1 'node.env - npm is not found!'
 
     (   set -e
-        # shellcheck disable=SC2030
-        PATH="$(npm bin):$PATH"
-        export PATH
 
         build_msg INSTALL "npm install $NPM_PACKAGES"
         # shellcheck disable=SC2086

requirements-dev.txt

@@ -6,9 +6,9 @@ pylint==2.15.9
 splinter==0.19.0
 transifex-client==0.14.3; python_version < '3.10'
 transifex-client==0.12.5; python_version == '3.10'
-selenium==4.7.2
+selenium==4.8.3
 twine==4.0.2
-Pallets-Sphinx-Themes==2.0.2
+Pallets-Sphinx-Themes==2.0.3
 docutils==0.18
 Sphinx==5.3.0
 sphinx-issues==3.0.1

searx/data/useragents.json

@@ -1,9 +1,9 @@
 {
     "versions": [
-        "108.0.1",
-        "108.0",
-        "107.0.1",
-        "107.0"
+        "111.0.1",
+        "111.0",
+        "110.0.1",
+        "110.0"
     ],
     "os": [
         "Windows NT 10.0; WOW64",

searx/data/wikidata_units.json

@@ -153,7 +153,6 @@
     "Q107164998": "cd mm²/m²",
     "Q107210119": "g/s",
     "Q107210344": "mg/s",
-    "Q107213614": "kJ/100g",
     "Q107226391": "cm⁻¹",
     "Q1072404": "K",
     "Q107244316": "mm⁻¹",
@@ -208,6 +207,7 @@
     "Q1091257": "tex",
     "Q1092296": "a",
     "Q110143852": "Ω cm",
+    "Q110143896": "cm³/g",
     "Q1104069": "$",
     "Q11061003": "μm²",
     "Q11061005": "nm²",
@@ -216,23 +216,29 @@
     "Q1137675": "cr",
     "Q114002440": "𒄀",
     "Q114002534": "𒃻",
-    "Q114002568": "𒂠",
     "Q114002639": "𒈨𒊑",
-    "Q114002688": "𒋗𒋛",
-    "Q114002734": "𒊺",
     "Q114002796": "𒂆",
-    "Q114002897": "𒊬",
     "Q114002930": "𒀺",
     "Q114002955": "𒀹𒃷",
     "Q114002974": "𒃷",
-    "Q114002998": "𒁓",
-    "Q114018694": "𒄥",
-    "Q114018781": "𒁀𒌷𒂵",
     "Q1140444": "Zb",
     "Q1140577": "Yb",
     "Q114589269": "A",
     "Q1152074": "Pb",
     "Q1152323": "Tb",
+    "Q115277430": "QB",
+    "Q115280832": "RB",
+    "Q115359862": "qg",
+    "Q115359863": "rg",
+    "Q115359865": "Rg",
+    "Q115359866": "Qg",
+    "Q115359910": "Rm",
+    "Q115533751": "rm",
+    "Q115533764": "qm",
+    "Q115533776": "Qm",
+    "Q116432446": "ᵐ",
+    "Q116432563": "ˢ",
+    "Q116443090": "ʰ",
     "Q1165799": "mil",
     "Q11776930": "Mg",
     "Q11830636": "psf",
@@ -258,7 +264,7 @@
     "Q12269122": "YB/s",
     "Q12269308": "Zb/s",
     "Q12269309": "ZB/s",
-    "Q1238720": "vols",
+    "Q1238720": "vols.",
     "Q1247300": "cm H₂O",
     "Q12714022": "cwt",
     "Q12789864": "GeV",
@@ -272,7 +278,6 @@
     "Q1322380": "Ts",
     "Q1323615": "oz t",
     "Q132643": "kr",
-    "Q133011": "Ls",
     "Q13400897": "g",
     "Q13479685": "mm H2O",
     "Q1351253": "Eib",
@@ -300,7 +305,6 @@
     "Q14914907": "th",
     "Q14916719": "Gpc",
     "Q14923662": "Pm³",
-    "Q1511773": "LSd",
     "Q15120301": "l atm",
     "Q1542309": "xu",
     "Q1545979": "ft³",
@@ -322,7 +326,6 @@
     "Q17255465": "v_P",
     "Q173117": "R$",
     "Q1741429": "kpm",
-    "Q174467": "Lm",
     "Q174728": "cm",
     "Q174789": "mm",
     "Q175821": "μm",
@@ -346,13 +349,11 @@
     "Q182429": "m/s",
     "Q1826195": "dl",
     "Q18413919": "cm/s",
-    "Q184172": "F",
     "Q185078": "a",
     "Q185153": "erg",
     "Q185648": "Torr",
     "Q185759": "span",
     "Q1872619": "zs",
-    "Q189097": "₧",
     "Q190095": "Gy",
     "Q19017495": "mm²",
     "Q190951": "S$",
@@ -407,7 +408,7 @@
     "Q211256": "mi/h",
     "Q21154419": "PD",
     "Q211580": "BTU (th)",
-    "Q212120": "A h",
+    "Q212120": "A⋅h",
     "Q213005": "G$",
     "Q2140397": "in³",
     "Q214377": "ell",
@@ -447,7 +448,6 @@
     "Q23931040": "dam²",
     "Q23931103": "nmi²",
     "Q240468": "syr£",
-    "Q2414435": "$b.",
     "Q242988": "Lib$",
     "Q2438073": "ag",
     "Q2448803": "mV",
@@ -565,7 +565,7 @@
     "Q3773454": "Mpc",
     "Q3815076": "Kib",
     "Q3833309": "£",
-    "Q3858002": "mA h",
+    "Q3858002": "mA⋅h",
     "Q3867152": "ft/s²",
     "Q389062": "Tib",
     "Q3902688": "pl",
@@ -626,6 +626,8 @@
     "Q53393868": "GJ",
     "Q53393886": "PJ",
     "Q53393890": "EJ",
+    "Q53393893": "ZJ",
+    "Q53393898": "YJ",
     "Q53448786": "yHz",
     "Q53448790": "zHz",
     "Q53448794": "fHz",
@@ -639,6 +641,7 @@
     "Q53448826": "hHz",
     "Q53448828": "yJ",
     "Q53448832": "zJ",
+    "Q53448835": "fJ",
     "Q53448842": "pJ",
     "Q53448844": "nJ",
     "Q53448847": "μJ",
@@ -701,6 +704,7 @@
     "Q53951982": "Mt",
     "Q53952048": "kt",
     "Q54006645": "ZWb",
+    "Q54081354": "ZT",
     "Q54081925": "ZSv",
     "Q54082468": "ZS",
     "Q54083144": "ZΩ",
@@ -725,7 +729,6 @@
     "Q56157046": "nmol",
     "Q56157048": "pmol",
     "Q56160603": "fmol",
-    "Q56302633": "UM",
     "Q56317622": "Q_P",
     "Q56318907": "kbar",
     "Q56349362": "Bs.S",
@@ -1202,10 +1205,10 @@
     "Q11570": "kg",
     "Q11573": "m",
     "Q11574": "s",
+    "Q11579": "K",
     "Q11582": "L",
     "Q12129": "pc",
     "Q12438": "N",
-    "Q16068": "DM",
     "Q1811": "AU",
     "Q20764": "Ma",
     "Q2101": "e",

searx/engines/bing.py

@@ -52,8 +52,7 @@ def request(query, params):
         offset=offset)
 
     params['url'] = base_url + search_path
-    params['headers']['User-Agent'] = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
-                                       '(KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36')
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
 
     return params
 
@@ -82,11 +81,13 @@ def response(resp):
     for result in eval_xpath(dom, '//li[@class="b_algo"]'):
         link = eval_xpath(result, './/h2/a')[0]
         url = link.attrib.get('href')
+        pretty_url = extract_text(eval_xpath(result, './/cite'))
         title = extract_text(link)
         content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
+                        'pretty_url': pretty_url,
                         'title': title,
                         'content': content})
 

searx/engines/duckduckgo.py

@@ -73,7 +73,9 @@ def request(query, params):
     # link again and again ..
 
     params['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
+    params['headers']['Origin'] = 'https://lite.duckduckgo.com'
     params['headers']['Referer'] = 'https://lite.duckduckgo.com/'
+    params['headers']['User-Agent'] = 'Mozilla/5.0'
 
     # initial page does not have an offset
     if params['pageno'] == 2:

searx/engines/google.py

@@ -112,7 +112,7 @@ filter_mapping = {
 results_xpath = '//div[contains(@class, "MjjYud")]'
 title_xpath = './/h3[1]'
 href_xpath = './/a/@href'
-content_xpath = './/div[@data-content-feature=1]'
+content_xpath = './/div[@data-sncf]'
 results_xpath_mobile_ui = '//div[contains(@class, "g ")]'
 
 # google *sections* are no usual *results*, we ignore them

searx/engines/startpage.py

@@ -91,14 +91,13 @@ def get_sc_code(headers):
         dom = html.fromstring(resp.text)
 
         try:
-            href = eval_xpath(dom, '//input[@name="sc"]')[0].get('value')
+            sc_code = eval_xpath(dom, '//input[@name="sc"]')[0].get('value')
         except IndexError as exc:
             # suspend startpage API --> https://github.com/searxng/searxng/pull/695
             raise SearxEngineResponseException(
                 suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"
             ) from exc
 
-        sc_code = href[5:]
         sc_code_ts = time()
         logger.debug("new value is: %s", sc_code)
 

searx/plugins/search_operators.py

@@ -11,7 +11,11 @@ default_on = False
 
 def on_result(request, search, result):
     q = search.search_query.query
-    qs = shlex.split(q)
+    # WARN: shlex.quote is designed only for Unix shells and may be vulnerable
+    # to command injection on non-POSIX compliant shells (Windows)
+    # https://docs.python.org/3/library/shlex.html#shlex.quote
+    squote = shlex.quote(q)
+    qs = shlex.split(squote)
     spitems = [x.lower() for x in qs if ' ' in x]
     mitems = [x.lower() for x in qs if x.startswith('-')]
     siteitems = [x.lower() for x in qs if x.startswith('site:')]

searx/settings.yml

@@ -1291,7 +1291,7 @@ engines:
   - name : wiby
     engine : json_engine
     paging : True
-    search_url : https://wiby.me/json/?q={query}&o={pageno}0
+    search_url : https://wiby.me/json/?q={query}&p={pageno}
     url_query : URL
     title_query : Title
     content_query : Snippet

searx_extra/update/update_firefox_version.py

@@ -5,7 +5,7 @@ import requests
 import re
 from os.path import dirname, join
 from urllib.parse import urlparse, urljoin
-from distutils.version import LooseVersion, StrictVersion
+from packaging.version import Version, parse
 from lxml import html
 from searx import searx_dir
 
@@ -39,7 +39,7 @@ def fetch_firefox_versions():
             if path.startswith(RELEASE_PATH):
                 version = path[len(RELEASE_PATH):-1]
                 if NORMAL_REGEX.match(version):
-                    versions.append(LooseVersion(version))
+                    versions.append(Version(version))
 
         list.sort(versions, reverse=True)
         return versions
@@ -49,12 +49,12 @@ def fetch_firefox_last_versions():
     versions = fetch_firefox_versions()
 
     result = []
-    major_last = versions[0].version[0]
+    major_last = versions[0].major
     major_list = (major_last, major_last - 1)
     for version in versions:
-        major_current = version.version[0]
+        major_current = version.major
         if major_current in major_list:
-            result.append(version.vstring)
+            result.append(str(version))
 
     return result