Merge branch 'master' into feature/tag_privacy_violators

requirements-dev.txt

@@ -2,7 +2,7 @@ mock==4.0.3
 nose2[coverage_plugin]==0.12.0
 cov-core==1.15.0
 pycodestyle==2.9.1
-pylint==2.14.5
+pylint==2.15.0
 splinter==0.18.1
 transifex-client==0.14.3; python_version < '3.10'
 transifex-client==0.12.5; python_version == '3.10'

searx/engines/ipfs_search.py

@@ -0,0 +1,93 @@
+from urllib.parse import urlencode
+import json
+import re
+from datetime import datetime
+
+# config
+categories = ['general', 'images', 'music', 'videos']
+paging = True
+time_range_support = True
+
+# url
+base_url = 'https://api.ipfs-search.com/v1/'
+search_string = 'search?{query} first-seen:{time_range} metadata.Content-Type:({mime_type})&page={page} '
+
+
+mime_types_map = {
+    'general': "*",
+    'images': 'image*',
+    'music': 'audio*',
+    'videos': 'video*'
+}
+
+time_range_map = {'day': '[ now-24h\/h TO *]',
+                  'week': '[ now\/h-7d TO *]',
+                  'month': '[ now\/d-30d TO *]',
+                  'year': '[ now\/d-1y TO *]'}
+
+ipfs_url = 'https://gateway.ipfs.io/ipfs/{hash}'
+
+
+def request(query, params):
+    mime_type = mime_types_map.get(params['category'], '*')
+    time_range = time_range_map.get(params['time_range'], '*')
+    search_path = search_string.format(
+        query=urlencode({'q': query}),
+        time_range=time_range,
+        page=params['pageno'],
+        mime_type=mime_type)
+
+    params['url'] = base_url + search_path
+
+    return params
+
+
+def clean_html(text):
+    if not text:
+        return ""
+    return str(re.sub(re.compile('<.*?>'), '', text))
+
+
+def create_base_result(record):
+    url = ipfs_url.format(hash=record.get('hash'))
+    title = clean_html(record.get('title'))
+    published_date = datetime.strptime(record.get('first-seen'), '%Y-%m-%dT%H:%M:%SZ')
+    return {'url': url,
+            'title': title,
+            'publishedDate': published_date}
+
+
+def create_text_result(record):
+    result = create_base_result(record)
+    description = clean_html(record.get('description'))
+    result['description'] = description
+    return result
+
+
+def create_image_result(record):
+    result = create_base_result(record)
+    result['img_src'] = result['url']
+    result['template'] = 'images.html'
+    return result
+
+
+def create_video_result(record):
+    result = create_base_result(record)
+    result['thumbnail'] = ''
+    result['template'] = 'videos.html'
+    return result
+
+
+def response(resp):
+    api_results = json.loads(resp.text)
+    results = []
+    for result in api_results.get('hits', []):
+        mime_type = result.get('mimetype', 'text/plain')
+
+        if mime_type.startswith('image'):
+            results.append(create_image_result(result))
+        elif mime_type.startswith('video'):
+            results.append(create_video_result(result))
+        else:
+            results.append(create_text_result(result))
+    return results

searx/engines/spotify.py

@@ -23,6 +23,7 @@ categories = ['music']
 paging = True
 api_client_id = None
 api_client_secret = None
+timeout = 10.0
 
 # search-url
 url = 'https://api.spotify.com/'
@@ -40,9 +41,10 @@ def request(query, params):
 
     r = requests.post(
         'https://accounts.spotify.com/api/token',
+        timeout=timeout,
         data={'grant_type': 'client_credentials'},
         headers={'Authorization': 'Basic ' + base64.b64encode(
-            "{}:{}".format(api_client_id, api_client_secret).encode()
+            "{}:{}".format(api_client_id, api_client_secret).encode(),
         ).decode()}
     )
     j = loads(r.text)

searx/settings.yml

@@ -740,6 +740,13 @@ engines:
     shortcut: iv
     timeout : 5.0
     disabled : True
+  
+  - name: ipfs search
+    engine: ipfs_search
+    shortcut: ipfs
+    paging: True
+    timeout: 5.0
+    disabled: True
 
   - name: kickass
     engine : kickass

searx_extra/update/update_external_bangs.py

@@ -30,7 +30,7 @@ HTTP_COLON = 'http:'
 
 
 def get_bang_url():
-    response = requests.get(URL_BV1)
+    response = requests.get(URL_BV1, timeout=10.0)
     response.raise_for_status()
 
     r = RE_BANG_VERSION.findall(response.text)
@@ -38,7 +38,7 @@ def get_bang_url():
 
 
 def fetch_ddg_bangs(url):
-    response = requests.get(url)
+    response = requests.get(url, timeout=10.0)
     response.raise_for_status()
     return json.loads(response.content.decode())