Merge pull request #3009 from kvch/pick-qwant-changes

Pick Qwant changes

searx/data/engines_languages.json

@@ -25773,6 +25773,49 @@
     "zh-CN",
     "zh-HK"
   ],
+  "qwant videos": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
   "startpage": {
     "af": {
       "alias": "afrikaans"

searx/engines/qwant.py

@@ -1,15 +1,41 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
+# lint: pylint
- Qwant (Web, Images, News, Social)
+"""Qwant (Web, News, Images, Videos)
+
+This engine uses the Qwant API (https://api.qwant.com/v3). The API is
+undocumented but can be reverse engineered by reading the network log of
+https://www.qwant.com/ queries.
+
+This implementation is used by different qwant engines in the settings.yml::
+
+  - name: qwant
+    categories: general
+    ...
+  - name: qwant news
+    categories: news
+    ...
+  - name: qwant images
+    categories: images
+    ...
+  - name: qwant videos
+    categories: videos
+    ...
+
 """
 
-from datetime import datetime
+from datetime import (
+    datetime,
+    timedelta,
+)
 from json import loads
 from urllib.parse import urlencode
-from searx.utils import html_to_text, match_language
+from flask_babel import gettext
-from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
+
+from searx.utils import match_language
+from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
 
+
 # about
 about = {
     "website": 'https://www.qwant.com/',
@@ -25,98 +51,175 @@ categories = []
 paging = True
 supported_languages_url = about['website']
 
-category_to_keyword = {'general': 'web',
+category_to_keyword = {
-                       'images': 'images',
+    'general': 'web',
-                       'news': 'news'}
+    'news': 'news',
+    'images': 'images',
+    'videos': 'videos',
+}
 
 # search-url
-url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'
+url = 'https://api.qwant.com/v3/search/{keyword}?q={query}&count={count}&offset={offset}'
 
 
-# do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 10
+    """Qwant search request"""
+    keyword = category_to_keyword[categories[0]]
+    count = 10  # web: count must be equal to 10
 
-    if categories[0] and categories[0] in category_to_keyword:
+    if keyword == 'images':
-
+        count = 50
-        params['url'] = url.format(keyword=category_to_keyword[categories[0]],
+        offset = (params['pageno'] - 1) * count
-                                   query=urlencode({'q': query}),
+        # count + offset must be lower than 250
-                                   offset=offset)
+        offset = min(offset, 199)
     else:
-        params['url'] = url.format(keyword='web',
+        offset = (params['pageno'] - 1) * count
-                                   query=urlencode({'q': query}),
+        # count + offset must be lower than 50
-                                   offset=offset)
+        offset = min(offset, 40)
+
+    params['url'] = url.format(
+        keyword=keyword,
+        query=urlencode({'q': query}),
+        offset=offset,
+        count=count,
+    )
 
     # add language tag
-    if params['language'] != 'all':
+    if params['language'] == 'all':
-        language = match_language(params['language'], supported_languages, language_aliases)
+        params['url'] += '&locale=en_us'
+    else:
+        language = match_language(
+            params['language'],
+            # pylint: disable=undefined-variable
+            supported_languages,
+            language_aliases,
+        )
         params['url'] += '&locale=' + language.replace('-', '_').lower()
 
-    params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'
     params['raise_for_httperror'] = False
     return params
 
 
-# get response from search-request
 def response(resp):
+    """Get response from Qwant's search request"""
+    # pylint: disable=too-many-locals, too-many-branches, too-many-statements
+
+    keyword = category_to_keyword[categories[0]]
     results = []
 
-    # According to https://www.qwant.com/js/app.js
+    # load JSON result
-    if resp.status_code == 429:
+    search_results = loads(resp.text)
-        raise SearxEngineCaptchaException()
+    data = search_results.get('data', {})
+
+    # check for an API error
+    if search_results.get('status') != 'success':
+        msg = ",".join(data.get('message', ['unknown', ]))
+        raise SearxEngineAPIException('API error::' + msg)
 
     # raise for other errors
     raise_for_httperror(resp)
 
-    # load JSON result
+    if keyword == 'web':
-    search_results = loads(resp.text)
+        # The WEB query contains a list named 'mainline'.  This list can contain
-
+        # different result types (e.g. mainline[0]['type'] returns type of the
-    # check for an API error
+        # result items in mainline[0]['items']
-    if search_results.get('status') != 'success':
+        mainline = data.get('result', {}).get('items', {}).get('mainline', {})
-        raise SearxEngineAPIException('API error ' + str(search_results.get('error', '')))
+    else:
+        # Queries on News, Images and Videos do not have a list named 'mainline'
+        # in the response.  The result items are directly in the list
+        # result['items'].
+        mainline = data.get('result', {}).get('items', [])
+        mainline = [
+            {'type': keyword, 'items': mainline},
+        ]
 
     # return empty array if there are no results
-    if 'data' not in search_results:
+    if not mainline:
         return []
 
-    data = search_results.get('data', {})
+    for row in mainline:
 
-    res = data.get('result', {})
+        mainline_type = row.get('type', 'web')
+        if mainline_type == 'ads':
+            # ignore adds
+            continue
 
-    # parse results
+        mainline_items = row.get('items', [])
-    for result in res.get('items', {}):
+        for item in mainline_items:
 
-        title = html_to_text(result['title'])
+            title = item.get('title', None)
-        res_url = result['url']
+            res_url = item.get('url', None)
-        content = html_to_text(result['desc'])
 
-        if category_to_keyword.get(categories[0], '') == 'web':
+            if mainline_type == 'web':
-            results.append({'title': title,
+                content = item['desc']
-                            'content': content,
+                results.append({
-                            'url': res_url})
+                    'title': title,
+                    'url': res_url,
+                    'content': content,
+                })
 
-        elif category_to_keyword.get(categories[0], '') == 'images':
+            elif mainline_type == 'news':
-            thumbnail_src = result['thumbnail']
-            img_src = result['media']
-            results.append({'template': 'images.html',
-                            'url': res_url,
-                            'title': title,
-                            'content': '',
-                            'thumbnail_src': thumbnail_src,
-                            'img_src': img_src})
 
-        elif category_to_keyword.get(categories[0], '') == 'news':
+                pub_date = item['date']
-            published_date = datetime.fromtimestamp(result['date'], None)
+                if pub_date is not None:
-            media = result.get('media', [])
+                    pub_date = datetime.fromtimestamp(pub_date)
-            if len(media) > 0:
+                news_media = item.get('media', [])
-                img_src = media[0].get('pict', {}).get('url', None)
-            else:
                 img_src = None
-            results.append({'url': res_url,
+                if news_media:
-                            'title': title,
+                    img_src = news_media[0].get('pict', {}).get('url', None)
-                            'publishedDate': published_date,
+                results.append({
-                            'content': content,
+                    'title': title,
-                            'img_src': img_src})
+                    'url': res_url,
+                    'publishedDate': pub_date,
+                    'img_src': img_src,
+                })
+
+            elif mainline_type == 'images':
+                thumbnail = item['thumbnail']
+                img_src = item['media']
+                results.append({
+                    'title': title,
+                    'url': res_url,
+                    'template': 'images.html',
+                    'thumbnail_src': thumbnail,
+                    'img_src': img_src,
+                })
+
+            elif mainline_type == 'videos':
+                # some videos do not have a description: while qwant-video
+                # returns an empty string, such video from a qwant-web query
+                # miss the 'desc' key.
+                d, s, c = item.get('desc'), item.get('source'), item.get('channel')
+                content_parts = []
+                if d:
+                    content_parts.append(d)
+                if s:
+                    content_parts.append("%s: %s " % (gettext("Source"), s))
+                if c:
+                    content_parts.append("%s: %s " % (gettext("Channel"), c))
+                content = ' // '.join(content_parts)
+                length = item['duration']
+                if length is not None:
+                    length = timedelta(milliseconds=length)
+                pub_date = item['date']
+                if pub_date is not None:
+                    pub_date = datetime.fromtimestamp(pub_date)
+                thumbnail = item['thumbnail']
+                # from some locations (DE and others?) the s2 link do
+                # response a 'Please wait ..' but does not deliver the thumbnail
+                thumbnail = thumbnail.replace(
+                    'https://s2.qwant.com',
+                    'https://s1.qwant.com', 1
+                )
+                results.append({
+                    'title': title,
+                    'url': res_url,
+                    'content': content,
+                    'publishedDate': pub_date,
+                    'thumbnail': thumbnail,
+                    'template': 'videos.html',
+                    'length': length,
+                })
 
     return results
 

searx/settings.yml

@@ -988,18 +988,26 @@ engines:
     additional_tests:
       rosebud: *test_rosebud
 
-  - name : qwant images
-    engine : qwant
-    shortcut : qwi
-    categories : images
-    network: qwant
-
   - name : qwant news
     engine : qwant
     shortcut : qwn
     categories : news
     network: qwant
 
+  - name: qwant images
+    engine: qwant
+    shortcut: qwi
+    categories: images
+    disabled: True
+    network: qwant
+
+  - name: qwant videos
+    engine: qwant
+    shortcut: qwv
+    categories: videos
+    disabled: True
+    network: qwant
+
 #  - name: library
 #    engine: recoll
 #    shortcut: lib