mirror of https://github.com/searx/searx
[enh] currency_convert engine: user can write "1 dollars in euros" instead of "1 USD in EUR".
The currency names are fetched wikidata and store into a static file : searx/data/currencies.json This file is loaded when the currency_converter is loaded. A database is perhaps more appropiated.
This commit is contained in:
parent
3683b35877
commit
5525625dae
File diff suppressed because it is too large
Load Diff
|
@ -1,11 +1,37 @@
|
|||
from datetime import datetime
|
||||
import re
|
||||
import os
|
||||
import json
|
||||
import unicodedata
|
||||
|
||||
|
||||
categories = []
|
||||
url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
|
||||
weight = 100
|
||||
|
||||
parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I) # noqa
|
||||
parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W*in?\W*([^\.]+)\W*$', re.I) # noqa
|
||||
|
||||
db = 1
|
||||
|
||||
|
||||
def normalize_name(name):
|
||||
name = name.lower().replace('-', ' ')
|
||||
name = re.sub(' +', ' ', name)
|
||||
return unicodedata.normalize('NFKD', u"" + name).lower()
|
||||
|
||||
|
||||
def name_to_iso4217(name):
|
||||
global db
|
||||
|
||||
name = normalize_name(name)
|
||||
currencies = db['names'].get(name, [name])
|
||||
return currencies[0]
|
||||
|
||||
|
||||
def iso4217_to_name(iso4217, language):
|
||||
global db
|
||||
|
||||
return db['iso4217'].get(iso4217, {}).get(language, iso4217)
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
@ -16,6 +42,8 @@ def request(query, params):
|
|||
|
||||
ammount, from_currency, to_currency = m.groups()
|
||||
ammount = float(ammount)
|
||||
from_currency = name_to_iso4217(from_currency.strip())
|
||||
to_currency = name_to_iso4217(to_currency.strip())
|
||||
|
||||
q = (from_currency + to_currency).upper()
|
||||
|
||||
|
@ -23,6 +51,8 @@ def request(query, params):
|
|||
params['ammount'] = ammount
|
||||
params['from'] = from_currency
|
||||
params['to'] = to_currency
|
||||
params['from_name'] = iso4217_to_name(from_currency, 'en')
|
||||
params['to_name'] = iso4217_to_name(to_currency, 'en')
|
||||
|
||||
return params
|
||||
|
||||
|
@ -35,12 +65,14 @@ def response(resp):
|
|||
except:
|
||||
return results
|
||||
|
||||
answer = '{0} {1} = {2} {3} (1 {1} = {4} {3})'.format(
|
||||
answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(
|
||||
resp.search_params['ammount'],
|
||||
resp.search_params['from'],
|
||||
resp.search_params['ammount'] * conversion_rate,
|
||||
resp.search_params['to'],
|
||||
conversion_rate
|
||||
conversion_rate,
|
||||
resp.search_params['from_name'],
|
||||
resp.search_params['to_name'],
|
||||
)
|
||||
|
||||
now_date = datetime.now().strftime('%Y%m%d')
|
||||
|
@ -55,3 +87,15 @@ def response(resp):
|
|||
results.append({'answer': answer, 'url': url})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def load():
|
||||
global db
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
json_data = open(current_dir + "/../data/currencies.json").read()
|
||||
|
||||
db = json.loads(json_data)
|
||||
|
||||
|
||||
load()
|
||||
|
|
|
@ -27,9 +27,11 @@ class TestCurrencyConvertEngine(SearxTestCase):
|
|||
|
||||
def test_response(self):
|
||||
dicto = defaultdict(dict)
|
||||
dicto['ammount'] = 10
|
||||
dicto['ammount'] = float(10)
|
||||
dicto['from'] = "EUR"
|
||||
dicto['to'] = "USD"
|
||||
dicto['from_name'] = "euro"
|
||||
dicto['to_name'] = "United States dollar"
|
||||
response = mock.Mock(text='a,b,c,d', search_params=dicto)
|
||||
self.assertEqual(currency_convert.response(response), [])
|
||||
|
||||
|
@ -38,7 +40,7 @@ class TestCurrencyConvertEngine(SearxTestCase):
|
|||
results = currency_convert.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]['answer'], '10 EUR = 5.0 USD (1 EUR = 0.5 USD)')
|
||||
self.assertEqual(results[0]['answer'], '10.0 EUR = 5.0 USD, 1 EUR (euro) = 0.5 USD (United States dollar)')
|
||||
now_date = datetime.now().strftime('%Y%m%d')
|
||||
self.assertEqual(results[0]['url'], 'https://finance.yahoo.com/currency/converter-results/' +
|
||||
now_date + '/10-eur-to-usd.html')
|
||||
now_date + '/10.0-eur-to-usd.html')
|
||||
|
|
|
@ -0,0 +1,161 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
import string
|
||||
from urllib import urlencode
|
||||
from requests import get
|
||||
|
||||
languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}
|
||||
|
||||
url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)
|
||||
url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
|
||||
url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
|
||||
|
||||
wmflabs_queries = [
|
||||
'CLAIM[31:8142]', # all devise
|
||||
]
|
||||
|
||||
db = {
|
||||
'iso4217' : {
|
||||
},
|
||||
'names' : {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def remove_accents(data):
|
||||
return unicodedata.normalize('NFKD', data).lower()
|
||||
|
||||
|
||||
def normalize_name(name):
|
||||
return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' '))
|
||||
|
||||
|
||||
def add_currency_name(name, iso4217):
|
||||
global db
|
||||
|
||||
db_names = db['names']
|
||||
|
||||
|
||||
if not isinstance(iso4217, basestring):
|
||||
print "problem", name, iso4217
|
||||
return
|
||||
|
||||
name = normalize_name(name)
|
||||
|
||||
if name == '':
|
||||
print "name empty", iso4217
|
||||
return
|
||||
|
||||
iso4217_set = db_names.get(name, None)
|
||||
if iso4217_set is not None and iso4217 not in iso4217_set:
|
||||
db_names[name].append(iso4217)
|
||||
else:
|
||||
db_names[name] = [ iso4217 ]
|
||||
|
||||
|
||||
def add_currency_label(label, iso4217, language):
|
||||
global db
|
||||
|
||||
db['iso4217'][iso4217] = db['iso4217'].get(iso4217, {})
|
||||
db['iso4217'][iso4217][language] = label
|
||||
|
||||
|
||||
def get_property_value(data, name):
|
||||
prop = data.get('claims', {}).get(name, {})
|
||||
if len(prop) == 0:
|
||||
return None
|
||||
|
||||
value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
|
||||
if value == '':
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def parse_currency(data):
|
||||
iso4217 = get_property_value(data, 'P498')
|
||||
|
||||
if iso4217 is not None:
|
||||
unit = get_property_value(data, 'P558')
|
||||
if unit is not None:
|
||||
add_currency_name(unit, iso4217)
|
||||
|
||||
labels = data.get('labels', {})
|
||||
for language in languages:
|
||||
name = labels.get(language, {}).get('value', None)
|
||||
if name != None:
|
||||
add_currency_name(name, iso4217)
|
||||
add_currency_label(name, iso4217, language)
|
||||
|
||||
aliases = data.get('aliases', {})
|
||||
for language in aliases:
|
||||
for i in range(0, len(aliases[language])):
|
||||
alias = aliases[language][i].get('value', None)
|
||||
add_currency_name(alias, iso4217)
|
||||
|
||||
|
||||
def fetch_data(wikidata_ids):
|
||||
url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)}))
|
||||
htmlresponse = get(url)
|
||||
jsonresponse = json.loads(htmlresponse.content)
|
||||
entities = jsonresponse.get('entities', {})
|
||||
|
||||
for pname in entities:
|
||||
pvalue = entities.get(pname)
|
||||
parse_currency(pvalue)
|
||||
|
||||
|
||||
def add_q(i):
|
||||
return "Q" + str(i)
|
||||
|
||||
|
||||
def fetch_data_batch(wikidata_ids):
|
||||
while len(wikidata_ids) > 0:
|
||||
if len(wikidata_ids) > 50:
|
||||
fetch_data(wikidata_ids[0:49])
|
||||
wikidata_ids = wikidata_ids[50:]
|
||||
else:
|
||||
fetch_data(wikidata_ids)
|
||||
wikidata_ids = []
|
||||
|
||||
|
||||
def wdq_query(query):
|
||||
url = url_wmflabs_template + query
|
||||
htmlresponse = get(url)
|
||||
jsonresponse = json.loads(htmlresponse.content)
|
||||
qlist = map(add_q, jsonresponse.get('items', {}))
|
||||
error = jsonresponse.get('status', {}).get('error', None)
|
||||
if error != None and error != 'OK':
|
||||
print "error for query '" + query + "' :" + error
|
||||
|
||||
fetch_data_batch(qlist)
|
||||
|
||||
|
||||
def wd_query(query, offset=0):
|
||||
qlist = []
|
||||
|
||||
url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))
|
||||
htmlresponse = get(url)
|
||||
jsonresponse = json.loads(htmlresponse.content)
|
||||
for r in jsonresponse.get('query', {}).get('search', {}):
|
||||
qlist.append(r.get('title', ''))
|
||||
fetch_data_batch(qlist)
|
||||
|
||||
## fetch ##
|
||||
for q in wmflabs_queries:
|
||||
wdq_query(q)
|
||||
|
||||
# static
|
||||
add_currency_name(u"euro", 'EUR')
|
||||
add_currency_name(u"euros", 'EUR')
|
||||
add_currency_name(u"dollar", 'USD')
|
||||
add_currency_name(u"dollars", 'USD')
|
||||
add_currency_name(u"peso", 'MXN')
|
||||
add_currency_name(u"pesos", 'MXN')
|
||||
|
||||
# write
|
||||
f = open("currencies.json", "wb")
|
||||
json.dump(db, f, indent=4, encoding="utf-8")
|
||||
f.close()
|
Loading…
Reference in New Issue