mirror of https://github.com/searx/searx
Merge branch 'master' into feature/tag_privacy_violators
This commit is contained in:
commit
feb5fbadfc
|
@ -2,7 +2,7 @@ mock==4.0.3
|
||||||
nose2[coverage_plugin]==0.12.0
|
nose2[coverage_plugin]==0.12.0
|
||||||
cov-core==1.15.0
|
cov-core==1.15.0
|
||||||
pycodestyle==2.9.1
|
pycodestyle==2.9.1
|
||||||
pylint==2.14.5
|
pylint==2.15.0
|
||||||
splinter==0.18.1
|
splinter==0.18.1
|
||||||
transifex-client==0.14.3; python_version < '3.10'
|
transifex-client==0.14.3; python_version < '3.10'
|
||||||
transifex-client==0.12.5; python_version == '3.10'
|
transifex-client==0.12.5; python_version == '3.10'
|
||||||
|
|
|
@ -0,0 +1,93 @@
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
# config
|
||||||
|
categories = ['general', 'images', 'music', 'videos']
|
||||||
|
paging = True
|
||||||
|
time_range_support = True
|
||||||
|
|
||||||
|
# url
|
||||||
|
base_url = 'https://api.ipfs-search.com/v1/'
|
||||||
|
search_string = 'search?{query} first-seen:{time_range} metadata.Content-Type:({mime_type})&page={page} '
|
||||||
|
|
||||||
|
|
||||||
|
mime_types_map = {
|
||||||
|
'general': "*",
|
||||||
|
'images': 'image*',
|
||||||
|
'music': 'audio*',
|
||||||
|
'videos': 'video*'
|
||||||
|
}
|
||||||
|
|
||||||
|
time_range_map = {'day': '[ now-24h\/h TO *]',
|
||||||
|
'week': '[ now\/h-7d TO *]',
|
||||||
|
'month': '[ now\/d-30d TO *]',
|
||||||
|
'year': '[ now\/d-1y TO *]'}
|
||||||
|
|
||||||
|
ipfs_url = 'https://gateway.ipfs.io/ipfs/{hash}'
|
||||||
|
|
||||||
|
|
||||||
|
def request(query, params):
|
||||||
|
mime_type = mime_types_map.get(params['category'], '*')
|
||||||
|
time_range = time_range_map.get(params['time_range'], '*')
|
||||||
|
search_path = search_string.format(
|
||||||
|
query=urlencode({'q': query}),
|
||||||
|
time_range=time_range,
|
||||||
|
page=params['pageno'],
|
||||||
|
mime_type=mime_type)
|
||||||
|
|
||||||
|
params['url'] = base_url + search_path
|
||||||
|
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
def clean_html(text):
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
return str(re.sub(re.compile('<.*?>'), '', text))
|
||||||
|
|
||||||
|
|
||||||
|
def create_base_result(record):
|
||||||
|
url = ipfs_url.format(hash=record.get('hash'))
|
||||||
|
title = clean_html(record.get('title'))
|
||||||
|
published_date = datetime.strptime(record.get('first-seen'), '%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
return {'url': url,
|
||||||
|
'title': title,
|
||||||
|
'publishedDate': published_date}
|
||||||
|
|
||||||
|
|
||||||
|
def create_text_result(record):
|
||||||
|
result = create_base_result(record)
|
||||||
|
description = clean_html(record.get('description'))
|
||||||
|
result['description'] = description
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def create_image_result(record):
|
||||||
|
result = create_base_result(record)
|
||||||
|
result['img_src'] = result['url']
|
||||||
|
result['template'] = 'images.html'
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def create_video_result(record):
|
||||||
|
result = create_base_result(record)
|
||||||
|
result['thumbnail'] = ''
|
||||||
|
result['template'] = 'videos.html'
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def response(resp):
|
||||||
|
api_results = json.loads(resp.text)
|
||||||
|
results = []
|
||||||
|
for result in api_results.get('hits', []):
|
||||||
|
mime_type = result.get('mimetype', 'text/plain')
|
||||||
|
|
||||||
|
if mime_type.startswith('image'):
|
||||||
|
results.append(create_image_result(result))
|
||||||
|
elif mime_type.startswith('video'):
|
||||||
|
results.append(create_video_result(result))
|
||||||
|
else:
|
||||||
|
results.append(create_text_result(result))
|
||||||
|
return results
|
|
@ -23,6 +23,7 @@ categories = ['music']
|
||||||
paging = True
|
paging = True
|
||||||
api_client_id = None
|
api_client_id = None
|
||||||
api_client_secret = None
|
api_client_secret = None
|
||||||
|
timeout = 10.0
|
||||||
|
|
||||||
# search-url
|
# search-url
|
||||||
url = 'https://api.spotify.com/'
|
url = 'https://api.spotify.com/'
|
||||||
|
@ -40,9 +41,10 @@ def request(query, params):
|
||||||
|
|
||||||
r = requests.post(
|
r = requests.post(
|
||||||
'https://accounts.spotify.com/api/token',
|
'https://accounts.spotify.com/api/token',
|
||||||
|
timeout=timeout,
|
||||||
data={'grant_type': 'client_credentials'},
|
data={'grant_type': 'client_credentials'},
|
||||||
headers={'Authorization': 'Basic ' + base64.b64encode(
|
headers={'Authorization': 'Basic ' + base64.b64encode(
|
||||||
"{}:{}".format(api_client_id, api_client_secret).encode()
|
"{}:{}".format(api_client_id, api_client_secret).encode(),
|
||||||
).decode()}
|
).decode()}
|
||||||
)
|
)
|
||||||
j = loads(r.text)
|
j = loads(r.text)
|
||||||
|
|
|
@ -740,6 +740,13 @@ engines:
|
||||||
shortcut: iv
|
shortcut: iv
|
||||||
timeout : 5.0
|
timeout : 5.0
|
||||||
disabled : True
|
disabled : True
|
||||||
|
|
||||||
|
- name: ipfs search
|
||||||
|
engine: ipfs_search
|
||||||
|
shortcut: ipfs
|
||||||
|
paging: True
|
||||||
|
timeout: 5.0
|
||||||
|
disabled: True
|
||||||
|
|
||||||
- name: kickass
|
- name: kickass
|
||||||
engine : kickass
|
engine : kickass
|
||||||
|
|
|
@ -30,7 +30,7 @@ HTTP_COLON = 'http:'
|
||||||
|
|
||||||
|
|
||||||
def get_bang_url():
|
def get_bang_url():
|
||||||
response = requests.get(URL_BV1)
|
response = requests.get(URL_BV1, timeout=10.0)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
r = RE_BANG_VERSION.findall(response.text)
|
r = RE_BANG_VERSION.findall(response.text)
|
||||||
|
@ -38,7 +38,7 @@ def get_bang_url():
|
||||||
|
|
||||||
|
|
||||||
def fetch_ddg_bangs(url):
|
def fetch_ddg_bangs(url):
|
||||||
response = requests.get(url)
|
response = requests.get(url, timeout=10.0)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return json.loads(response.content.decode())
|
return json.loads(response.content.decode())
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue