Merge pull request #973 from MarcAbonce/languages

Remove 'all' option from search languages
This commit is contained in:
Adam Tauber 2017-12-06 14:34:26 +01:00 committed by GitHub
commit 49b845051f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 166 additions and 249 deletions

View File

@ -26,7 +26,7 @@ xpath_results = '//ul[@class="mw-search-results"]/li'
xpath_link = './/div[@class="mw-search-result-heading"]/a' xpath_link = './/div[@class="mw-search-result-heading"]/a'
# cut 'en' from 'en_US', 'de' from 'de_CH', and so on # cut 'en' from 'en-US', 'de' from 'de-CH', and so on
def locale_to_lang_code(locale): def locale_to_lang_code(locale):
if locale.find('-') >= 0: if locale.find('-') >= 0:
locale = locale.split('-')[0] locale = locale.split('-')[0]
@ -36,7 +36,7 @@ def locale_to_lang_code(locale):
# wikis for some languages were moved off from the main site, we need to make # wikis for some languages were moved off from the main site, we need to make
# requests to correct URLs to be able to get results in those languages # requests to correct URLs to be able to get results in those languages
lang_urls = { lang_urls = {
'all': { 'en': {
'base': 'https://wiki.archlinux.org', 'base': 'https://wiki.archlinux.org',
'search': '/index.php?title=Special:Search&offset={offset}&{query}' 'search': '/index.php?title=Special:Search&offset={offset}&{query}'
}, },
@ -67,7 +67,7 @@ lang_urls = {
def get_lang_urls(language): def get_lang_urls(language):
if language in lang_urls: if language in lang_urls:
return lang_urls[language] return lang_urls[language]
return lang_urls['all'] return lang_urls['en']
# Language names to build search requests for # Language names to build search requests for

View File

@ -32,10 +32,7 @@ search_string = 'search?{query}&first={offset}'
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1 offset = (params['pageno'] - 1) * 10 + 1
if params['language'] != 'all': lang = params['language'].split('-')[0].upper()
lang = params['language'].split('-')[0].upper()
else:
lang = 'EN'
query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8') query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')

View File

@ -71,10 +71,7 @@ def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1 offset = (params['pageno'] - 1) * 10 + 1
if params['language'] == 'all': language = params['language']
language = 'en-US'
else:
language = params['language']
params['url'] = _get_url(query, language, offset, params['time_range']) params['url'] = _get_url(query, language, offset, params['time_range'])

View File

@ -32,10 +32,7 @@ supported_languages_url = 'https://api.dailymotion.com/languages'
# do search-request # do search-request
def request(query, params): def request(query, params):
if params['language'] == 'all': locale = params['language']
locale = 'en-US'
else:
locale = params['language']
params['url'] = search_url.format( params['url'] = search_url.format(
query=urlencode({'search': query, 'localization': locale}), query=urlencode({'search': query, 'localization': locale}),

View File

@ -44,9 +44,7 @@ content_xpath = './/a[@class="result__snippet"]'
# match query's language to a region code that duckduckgo will accept # match query's language to a region code that duckduckgo will accept
def get_region_code(lang, lang_list=None): def get_region_code(lang, lang_list=None):
# custom fixes for languages # custom fixes for languages
if lang == 'all': if lang[:2] == 'ja':
region_code = None
elif lang[:2] == 'ja':
region_code = 'jp-jp' region_code = 'jp-jp'
elif lang[:2] == 'sl': elif lang[:2] == 'sl':
region_code = 'sl-sl' region_code = 'sl-sl'
@ -82,12 +80,8 @@ def request(query, params):
offset = (params['pageno'] - 1) * 30 offset = (params['pageno'] - 1) * 30
region_code = get_region_code(params['language']) region_code = get_region_code(params['language'])
if region_code: params['url'] = url.format(
params['url'] = url.format( query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
else:
params['url'] = url.format(
query=urlencode({'q': query}), offset=offset, dc_param=offset)
if params['time_range'] in time_range_dict: if params['time_range'] in time_range_dict:
params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

View File

@ -53,12 +53,8 @@ def request(query, params):
safesearch = params['safesearch'] - 1 safesearch = params['safesearch'] - 1
region_code = get_region_code(params['language'], lang_list=supported_languages) region_code = get_region_code(params['language'], lang_list=supported_languages)
if region_code: params['url'] = images_url.format(
params['url'] = images_url.format( query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
else:
params['url'] = images_url.format(
query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
return params return params

View File

@ -40,10 +40,7 @@ def request(query, params):
offset = (params['pageno'] - 1) * number_of_results + 1 offset = (params['pageno'] - 1) * number_of_results + 1
categorie = search_category.get(params['category'], 'web') categorie = search_category.get(params['category'], 'web')
if params['language'] == 'all': language = params['language'].split('-')[0]
language = 'en'
else:
language = params['language'].split('_')[0]
# if language is not supported, put it in english # if language is not supported, put it in english
if language != 'en' and\ if language != 'en' and\

View File

@ -49,12 +49,9 @@ supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * number_of_results offset = (params['pageno'] - 1) * number_of_results
if params['language'] == 'all': language = params['language'].replace('-', '_').lower()
language = 'xx' if language.split('-')[0] != 'zh':
else: language = language.split('-')[0]
language = params['language'].replace('-', '_').lower()
if language.split('-')[0] != 'zh':
language = language.split('-')[0]
if params['safesearch'] >= 1: if params['safesearch'] >= 1:
safesearch = 1 safesearch = 1

View File

@ -165,7 +165,8 @@ def extract_text_from_dom(result, xpath):
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * 10 offset = (params['pageno'] - 1) * 10
if params['language'] == 'all': # temporary fix until a way of supporting en-US is found
if params['language'] == 'en-US':
params['language'] = 'en-GB' params['language'] = 'en-GB'
if params['language'][:2] == 'jv': if params['language'][:2] == 'jv':

View File

@ -50,9 +50,8 @@ def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}), params['url'] = search_url.format(query=urlencode({'q': query}),
search_options=urlencode(search_options)) search_options=urlencode(search_options))
if params['language'] != 'all': language_array = params['language'].lower().split('-')
language_array = params['language'].lower().split('-') params['url'] += '&lr=lang_' + language_array[0]
params['url'] += '&lr=lang_' + language_array[0]
return params return params

View File

@ -45,10 +45,7 @@ def request(query, params):
format_strings = list(Formatter().parse(base_url)) format_strings = list(Formatter().parse(base_url))
if params['language'] == 'all': language = params['language'].split('-')[0]
language = 'en'
else:
language = params['language'].split('-')[0]
# format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)] # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
if any(x[1] == 'language' for x in format_strings): if any(x[1] == 'language' for x in format_strings):

View File

@ -35,10 +35,9 @@ def request(query, params):
search_string.format(query=urlencode({'q': query}), search_string.format(query=urlencode({'q': query}),
limit=number_of_results) limit=number_of_results)
if params['language'] != 'all': language = params['language'].split('-')[0]
language = params['language'].split('_')[0] if language in supported_languages:
if language in supported_languages: params['url'] = params['url'] + "&lang=" + language
params['url'] = params['url'] + "&lang=" + language
# using searx User-Agent # using searx User-Agent
params['headers']['User-Agent'] = searx_useragent() params['headers']['User-Agent'] = searx_useragent()

View File

@ -44,18 +44,17 @@ def request(query, params):
query=urlencode({'q': query}), query=urlencode({'q': query}),
offset=offset) offset=offset)
# add language tag if specified # add language tag
if params['language'] != 'all': if params['language'] == 'no' or params['language'].startswith('no-'):
if params['language'] == 'no' or params['language'].startswith('no-'): params['language'] = params['language'].replace('no', 'nb', 1)
params['language'] = params['language'].replace('no', 'nb', 1) if params['language'].find('-') < 0:
if params['language'].find('-') < 0: # tries to get a country code from language
# tries to get a country code from language for lang in supported_languages:
for lang in supported_languages: lc = lang.split('-')
lc = lang.split('-') if params['language'] == lc[0]:
if params['language'] == lc[0]: params['language'] = lang
params['language'] = lang break
break params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
return params return params

View File

@ -45,9 +45,8 @@ def request(query, params):
params['data'] = {'query': query, params['data'] = {'query': query,
'startat': offset} 'startat': offset}
# set language if specified # set language
if params['language'] != 'all': params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
return params return params

View File

@ -48,7 +48,7 @@ def response(resp):
search_lang = 'Farsi' search_lang = 'Farsi'
elif resp.search_params['language'] == 'pt-BR': elif resp.search_params['language'] == 'pt-BR':
search_lang = 'Brazilian' search_lang = 'Brazilian'
elif resp.search_params['language'] != 'all': else:
search_lang = [lc[3] search_lang = [lc[3]
for lc in language_codes for lc in language_codes
if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]] if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]

View File

@ -35,10 +35,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
# do search-request # do search-request
def request(query, params): def request(query, params):
if params['language'] == 'all': if params['language'].split('-')[0] == 'no':
ui_language = 'browser'
region = 'browser'
elif params['language'].split('-')[0] == 'no':
region = 'nb-NO' region = 'nb-NO'
else: else:
region = params['language'] region = params['language']

View File

@ -37,12 +37,7 @@ timestamp_xpath = './/span[contains(@class,"_timestamp")]'
# do search-request # do search-request
def request(query, params): def request(query, params):
params['url'] = search_url + urlencode({'q': query}) params['url'] = search_url + urlencode({'q': query})
params['cookies']['lang'] = params['language'].split('-')[0]
# set language if specified
if params['language'] != 'all':
params['cookies']['lang'] = params['language'].split('-')[0]
else:
params['cookies']['lang'] = 'en'
return params return params

View File

@ -57,8 +57,6 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
def request(query, params): def request(query, params):
language = params['language'].split('-')[0] language = params['language'].split('-')[0]
if language == 'all':
language = 'en'
params['url'] = url_search.format( params['url'] = url_search.format(
query=urlencode({'label': query, 'language': language})) query=urlencode({'label': query, 'language': language}))
@ -71,8 +69,6 @@ def response(resp):
wikidata_ids = html.xpath(wikidata_ids_xpath) wikidata_ids = html.xpath(wikidata_ids_xpath)
language = resp.search_params['language'].split('-')[0] language = resp.search_params['language'].split('-')[0]
if language == 'all':
language = 'en'
# TODO: make requests asynchronous to avoid timeout when result_count > 1 # TODO: make requests asynchronous to avoid timeout when result_count > 1
for wikidata_id in wikidata_ids[:result_count]: for wikidata_id in wikidata_ids[:result_count]:

View File

@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
# set language in base_url # set language in base_url
def url_lang(lang): def url_lang(lang):
lang = lang.split('-')[0] lang = lang.split('-')[0]
if lang == 'all' or lang not in supported_languages: if lang not in supported_languages:
language = 'en' language = 'en'
else: else:
language = lang language = lang

View File

@ -51,9 +51,7 @@ def request(query, params):
limit=number_of_results, limit=number_of_results,
search_type=search_type) search_type=search_type)
# add language tag if specified params['url'] += '&lr=lang_' + params['language'].split('-')[0]
if params['language'] != 'all':
params['url'] += '&lr=lang_' + params['language'].split('-')[0]
return params return params

View File

@ -71,9 +71,7 @@ def _get_url(query, offset, language, time_range):
def _get_language(params): def _get_language(params):
if params['language'] == 'all': if params['language'][:2] == 'zh':
return 'en'
elif params['language'][:2] == 'zh':
if params['language'] == 'zh' or params['language'] == 'zh-CH': if params['language'] == 'zh' or params['language'] == 'zh-CH':
return 'szh' return 'szh'
else: else:

View File

@ -38,10 +38,7 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1 offset = (params['pageno'] - 1) * 10 + 1
if params['language'] == 'all': language = params['language'].split('-')[0]
language = 'en'
else:
language = params['language'].split('_')[0]
params['url'] = search_url.format(offset=offset, params['url'] = search_url.format(offset=offset,
query=urlencode({'p': query}), query=urlencode({'p': query}),

View File

@ -34,9 +34,7 @@ def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}), params['url'] = search_url.format(query=urlencode({'q': query}),
api_key=api_key) api_key=api_key)
# add language tag if specified params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
if params['language'] != 'all':
params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
return params return params

View File

@ -12,7 +12,6 @@ if version[0] == '3':
COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years
LANGUAGE_CODES = [l[0] for l in languages] LANGUAGE_CODES = [l[0] for l in languages]
LANGUAGE_CODES.append('all')
DISABLED = 0 DISABLED = 0
ENABLED = 1 ENABLED = 1
DOI_RESOLVERS = list(settings['doi_resolvers']) DOI_RESOLVERS = list(settings['doi_resolvers'])

View File

@ -73,11 +73,6 @@ class RawTextQuery(object):
if query_part[0] == ':': if query_part[0] == ':':
lang = query_part[1:].lower().replace('_', '-') lang = query_part[1:].lower().replace('_', '-')
# user may set a valid, yet not selectable language
if VALID_LANGUAGE_CODE.match(lang):
self.languages.append(lang)
parse_next = True
# check if any language-code is equal with # check if any language-code is equal with
# declared language-codes # declared language-codes
for lc in language_codes: for lc in language_codes:
@ -85,16 +80,25 @@ class RawTextQuery(object):
# if correct language-code is found # if correct language-code is found
# set it as new search-language # set it as new search-language
if lang == lang_id\ if (lang == lang_id
or lang_id.startswith(lang)\ or lang == lang_name
or lang == lang_name\ or lang == english_name
or lang == english_name\ or lang.replace('-', ' ') == country)\
or lang.replace('-', ' ') == country: and lang not in self.languages:
parse_next = True parse_next = True
self.languages.append(lang_id) lang_parts = lang_id.split('-')
# to ensure best match (first match is not necessarily the best one) if len(lang_parts) == 2:
if lang == lang_id: self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
break else:
self.languages.append(lang_id)
# to ensure best match (first match is not necessarily the best one)
if lang == lang_id:
break
# user may set a valid, yet not selectable language
if not self.languages and VALID_LANGUAGE_CODE.match(lang):
self.languages.append(lang)
parse_next = True
# this force a engine or category # this force a engine or category
if query_part[0] == '!' or query_part[0] == '?': if query_part[0] == '!' or query_part[0] == '?':

View File

@ -24,7 +24,7 @@ from flask_babel import gettext
import requests.exceptions import requests.exceptions
import searx.poolrequests as requests_lib import searx.poolrequests as requests_lib
from searx.engines import ( from searx.engines import (
categories, engines categories, engines, settings
) )
from searx.answerers import ask from searx.answerers import ask
from searx.utils import gen_useragent from searx.utils import gen_useragent
@ -220,6 +220,10 @@ def get_search_query_from_webapp(preferences, form):
else: else:
query_lang = preferences.get_value('language') query_lang = preferences.get_value('language')
# provides backwards compatibility for requests using old language default
if query_lang == 'all':
query_lang = settings['search']['language']
# check language # check language
if not VALID_LANGUAGE_CODE.match(query_lang): if not VALID_LANGUAGE_CODE.match(query_lang):
raise SearxParameterException('language', query_lang) raise SearxParameterException('language', query_lang)

View File

@ -5,7 +5,7 @@ general:
search: search:
safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
language : "all" language : "en-US"
server: server:
port : 8888 port : 8888

View File

@ -5,7 +5,7 @@ general:
search: search:
safe_search : 0 safe_search : 0
autocomplete : "" autocomplete : ""
language: "all" language: "en-US"
server: server:
port : 11111 port : 11111

View File

@ -13,7 +13,6 @@
<legend>{{ _('Search language') }}</legend> <legend>{{ _('Search language') }}</legend>
<p> <p>
<select name='language'> <select name='language'>
<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %} {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option> <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
{% endfor %} {% endfor %}

View File

@ -14,7 +14,6 @@
<legend>{{ _('Search language') }}</legend> <legend>{{ _('Search language') }}</legend>
<p> <p>
<select name='language'> <select name='language'>
<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %} {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option> <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
{% endfor %} {% endfor %}

View File

@ -3,10 +3,9 @@
{% else %} {% else %}
<select class="time_range custom-select form-control" id='language' name='language'> <select class="time_range custom-select form-control" id='language' name='language'>
{% endif %} {% endif %}
<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option> {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %} <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}> {{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }} </option>
</option> {% endfor %}
{% endfor %}
</select> </select>

View File

@ -187,7 +187,7 @@
</td> </td>
<th>{{ search_engine.name }}</th> <th>{{ search_engine.name }}</th>
<td class="name">{{ shortcuts[search_engine.name] }}</td> <td class="name">{{ shortcuts[search_engine.name] }}</td>
<td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td> <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>
<td>{{ support_toggle(search_engine.safesearch==True) }}</td> <td>{{ support_toggle(search_engine.safesearch==True) }}</td>
<td>{{ support_toggle(search_engine.time_range_support==True) }}</td> <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>
<td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td> <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
@ -197,7 +197,7 @@
<td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td> <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
<td>{{ support_toggle(search_engine.time_range_support==True) }}</td> <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>
<td>{{ support_toggle(search_engine.safesearch==True) }}</td> <td>{{ support_toggle(search_engine.safesearch==True) }}</td>
<td>{{ support_toggle(current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td> <td>{{ support_toggle(current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages) }}</td>
<td>{{ shortcuts[search_engine.name] }}</td> <td>{{ shortcuts[search_engine.name] }}</td>
<th>{{ search_engine.name }}</th> <th>{{ search_engine.name }}</th>
<td class="onoff-checkbox"> <td class="onoff-checkbox">

View File

@ -9,7 +9,6 @@
<legend>{{ _('Search language') }}</legend> <legend>{{ _('Search language') }}</legend>
<p> <p>
<select name='language'> <select name='language'>
<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %} {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option> <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
{% endfor %} {% endfor %}

View File

@ -630,8 +630,8 @@ def autocompleter():
if len(raw_results) <= 3 and completer: if len(raw_results) <= 3 and completer:
# get language from cookie # get language from cookie
language = request.preferences.get_value('language') language = request.preferences.get_value('language')
if not language or language == 'all': if not language:
language = 'en' language = settings['search']['language']
else: else:
language = language.split('-')[0] language = language.split('-')[0]
# run autocompletion # run autocompletion

View File

@ -4,7 +4,7 @@ from searx.engines import archlinux
from searx.testing import SearxTestCase from searx.testing import SearxTestCase
domains = { domains = {
'all': 'https://wiki.archlinux.org', 'en': 'https://wiki.archlinux.org',
'de': 'https://wiki.archlinux.de', 'de': 'https://wiki.archlinux.de',
'fr': 'https://wiki.archlinux.fr', 'fr': 'https://wiki.archlinux.fr',
'ja': 'https://wiki.archlinuxjp.org', 'ja': 'https://wiki.archlinuxjp.org',

View File

@ -10,17 +10,13 @@ class TestBingEngine(SearxTestCase):
query = u'test_query' query = u'test_query'
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['pageno'] = 0 dicto['pageno'] = 0
dicto['language'] = 'fr_FR' dicto['language'] = 'fr-FR'
params = bing.request(query.encode('utf-8'), dicto) params = bing.request(query.encode('utf-8'), dicto)
self.assertTrue('url' in params) self.assertTrue('url' in params)
self.assertTrue(query in params['url']) self.assertTrue(query in params['url'])
self.assertTrue('language%3AFR' in params['url']) self.assertTrue('language%3AFR' in params['url'])
self.assertTrue('bing.com' in params['url']) self.assertTrue('bing.com' in params['url'])
dicto['language'] = 'all'
params = bing.request(query.encode('utf-8'), dicto)
self.assertTrue('language' in params['url'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, bing.response, None) self.assertRaises(AttributeError, bing.response, None)
self.assertRaises(AttributeError, bing.response, []) self.assertRaises(AttributeError, bing.response, [])

View File

@ -19,10 +19,6 @@ class TestBingNewsEngine(SearxTestCase):
self.assertIn('bing.com', params['url']) self.assertIn('bing.com', params['url'])
self.assertIn('fr', params['url']) self.assertIn('fr', params['url'])
dicto['language'] = 'all'
params = bing_news.request(query, dicto)
self.assertIn('en', params['url'])
def test_no_url_in_request_year_time_range(self): def test_no_url_in_request_year_time_range(self):
dicto = defaultdict(dict) dicto = defaultdict(dict)
query = 'test_query' query = 'test_query'

View File

@ -18,10 +18,6 @@ class TestDailymotionEngine(SearxTestCase):
self.assertTrue('dailymotion.com' in params['url']) self.assertTrue('dailymotion.com' in params['url'])
self.assertTrue('fr' in params['url']) self.assertTrue('fr' in params['url'])
dicto['language'] = 'all'
params = dailymotion.request(query, dicto)
self.assertTrue('en' in params['url'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, dailymotion.response, None) self.assertRaises(AttributeError, dailymotion.response, None)
self.assertRaises(AttributeError, dailymotion.response, []) self.assertRaises(AttributeError, dailymotion.response, [])

View File

@ -15,7 +15,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):
dicto['is_test'] = True dicto['is_test'] = True
dicto['pageno'] = 1 dicto['pageno'] = 1
dicto['safesearch'] = 0 dicto['safesearch'] = 0
dicto['language'] = 'all' dicto['language'] = 'en-US'
params = duckduckgo_images.request(query, dicto) params = duckduckgo_images.request(query, dicto)
self.assertIn('url', params) self.assertIn('url', params)
self.assertIn(query, params['url']) self.assertIn(query, params['url'])

View File

@ -11,7 +11,7 @@ class TestFarooEngine(SearxTestCase):
query = 'test_query' query = 'test_query'
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['pageno'] = 1 dicto['pageno'] = 1
dicto['language'] = 'fr_FR' dicto['language'] = 'fr-FR'
dicto['category'] = 'general' dicto['category'] = 'general'
params = faroo.request(query, dicto) params = faroo.request(query, dicto)
self.assertIn('url', params) self.assertIn('url', params)
@ -20,11 +20,7 @@ class TestFarooEngine(SearxTestCase):
self.assertIn('en', params['url']) self.assertIn('en', params['url'])
self.assertIn('web', params['url']) self.assertIn('web', params['url'])
dicto['language'] = 'all' dicto['language'] = 'de-DE'
params = faroo.request(query, dicto)
self.assertIn('en', params['url'])
dicto['language'] = 'de_DE'
params = faroo.request(query, dicto) params = faroo.request(query, dicto)
self.assertIn('de', params['url']) self.assertIn('de', params['url'])

View File

@ -11,16 +11,11 @@ class TestGigablastEngine(SearxTestCase):
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['pageno'] = 0 dicto['pageno'] = 0
dicto['safesearch'] = 0 dicto['safesearch'] = 0
dicto['language'] = 'all' dicto['language'] = 'en-US'
params = gigablast.request(query, dicto) params = gigablast.request(query, dicto)
self.assertTrue('url' in params) self.assertTrue('url' in params)
self.assertTrue(query in params['url']) self.assertTrue(query in params['url'])
self.assertTrue('gigablast.com' in params['url']) self.assertTrue('gigablast.com' in params['url'])
self.assertTrue('xx' in params['url'])
dicto['language'] = 'en-US'
params = gigablast.request(query, dicto)
self.assertTrue('en' in params['url'])
self.assertFalse('en-US' in params['url']) self.assertFalse('en-US' in params['url'])
def test_response(self): def test_response(self):

View File

@ -26,7 +26,7 @@ class TestGoogleEngine(SearxTestCase):
self.assertIn('google.fr', params['url']) self.assertIn('google.fr', params['url'])
self.assertIn('fr', params['headers']['Accept-Language']) self.assertIn('fr', params['headers']['Accept-Language'])
dicto['language'] = 'all' dicto['language'] = 'en-US'
params = google.request(query, dicto) params = google.request(query, dicto)
self.assertIn('google.co', params['url']) self.assertIn('google.co', params['url'])
self.assertIn('en', params['headers']['Accept-Language']) self.assertIn('en', params['headers']['Accept-Language'])

View File

@ -12,18 +12,13 @@ class TestGoogleNewsEngine(SearxTestCase):
query = 'test_query' query = 'test_query'
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['pageno'] = 1 dicto['pageno'] = 1
dicto['language'] = 'fr_FR' dicto['language'] = 'fr-FR'
dicto['time_range'] = 'w' dicto['time_range'] = 'w'
params = google_news.request(query, dicto) params = google_news.request(query, dicto)
self.assertIn('url', params) self.assertIn('url', params)
self.assertIn(query, params['url']) self.assertIn(query, params['url'])
self.assertIn('fr', params['url']) self.assertIn('fr', params['url'])
dicto['language'] = 'all'
params = google_news.request(query, dicto)
self.assertIn('url', params)
self.assertNotIn('fr', params['url'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, google_news.response, None) self.assertRaises(AttributeError, google_news.response, None)
self.assertRaises(AttributeError, google_news.response, []) self.assertRaises(AttributeError, google_news.response, [])

View File

@ -18,10 +18,6 @@ class TestMediawikiEngine(SearxTestCase):
self.assertIn('wikipedia.org', params['url']) self.assertIn('wikipedia.org', params['url'])
self.assertIn('fr', params['url']) self.assertIn('fr', params['url'])
dicto['language'] = 'all'
params = mediawiki.request(query, dicto)
self.assertIn('en', params['url'])
mediawiki.base_url = "http://test.url/" mediawiki.base_url = "http://test.url/"
mediawiki.search_url = mediawiki.base_url +\ mediawiki.search_url = mediawiki.base_url +\
'w/api.php?action=query'\ 'w/api.php?action=query'\

View File

@ -19,10 +19,11 @@ class TestQwantEngine(SearxTestCase):
self.assertIn('qwant.com', params['url']) self.assertIn('qwant.com', params['url'])
self.assertIn('fr_fr', params['url']) self.assertIn('fr_fr', params['url'])
dicto['language'] = 'all' dicto['language'] = 'en-US'
qwant.categories = ['news'] qwant.categories = ['news']
params = qwant.request(query, dicto) params = qwant.request(query, dicto)
self.assertFalse('fr' in params['url']) self.assertFalse('fr' in params['url'])
self.assertIn('en_us', params['url'])
self.assertIn('news', params['url']) self.assertIn('news', params['url'])
qwant.supported_languages = ['en', 'fr-FR', 'fr-CA'] qwant.supported_languages = ['en', 'fr-FR', 'fr-CA']

View File

@ -21,10 +21,6 @@ class TestStartpageEngine(SearxTestCase):
self.assertIn('with_language', params['data']) self.assertIn('with_language', params['data'])
self.assertIn('lang_fr', params['data']['with_language']) self.assertIn('lang_fr', params['data']['with_language'])
dicto['language'] = 'all'
params = startpage.request(query, dicto)
self.assertNotIn('with_language', params['data'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, startpage.response, None) self.assertRaises(AttributeError, startpage.response, None)
self.assertRaises(AttributeError, startpage.response, []) self.assertRaises(AttributeError, startpage.response, [])

View File

@ -73,42 +73,6 @@ class TestSubtitleseekerEngine(SearxTestCase):
results = subtitleseeker.response(response) results = subtitleseeker.response(response)
self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/') self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')
html = """
<div class="boxRows">
<div class="boxRowsInner" style="width:600px;">
<img src="http://static.subtitleseeker.com/images/movie.gif"
style="width:16px; height:16px;" class="icon">
<a href="http://this.is.the.url/"
class="blue" title="Title subtitle" >
This is the Title
</a>
</div>
<div class="boxRowsInner f12b red" style="width:70px;">
1998
</div>
<div class="boxRowsInner grey-web f12" style="width:120px;">
<img src="http://static.subtitleseeker.com/images/basket_put.png"
style="width:16px; height:16px;" class="icon">
1039 Subs
</div>
<div class="boxRowsInner grey-web f10" style="width:130px;">
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
style="width:16px; height:16px;" class="icon">
1 hours ago
</div>
<div class="clear"></div>
</div>
"""
dicto['language'] = 'all'
response = mock.Mock(text=html, search_params=dicto)
results = subtitleseeker.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the Title')
self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
self.assertIn('1998', results[0]['content'])
self.assertIn('1039 Subs', results[0]['content'])
html = """ html = """
<div class="boxRows"> <div class="boxRows">
<div class="boxRowsInner" style="width:600px;"> <div class="boxRowsInner" style="width:600px;">

View File

@ -18,11 +18,6 @@ class TestSwisscowsEngine(SearxTestCase):
self.assertTrue('uiLanguage=de' in params['url']) self.assertTrue('uiLanguage=de' in params['url'])
self.assertTrue('region=de-DE' in params['url']) self.assertTrue('region=de-DE' in params['url'])
dicto['language'] = 'all'
params = swisscows.request(query, dicto)
self.assertTrue('uiLanguage=browser' in params['url'])
self.assertTrue('region=browser' in params['url'])
dicto['category'] = 'images' dicto['category'] = 'images'
params = swisscows.request(query, dicto) params = swisscows.request(query, dicto)
self.assertIn('image', params['url']) self.assertIn('image', params['url'])

View File

@ -20,12 +20,6 @@ class TestTwitterEngine(SearxTestCase):
self.assertIn('lang', params['cookies']) self.assertIn('lang', params['cookies'])
self.assertIn('fr', params['cookies']['lang']) self.assertIn('fr', params['cookies']['lang'])
dicto['language'] = 'all'
params = twitter.request(query, dicto)
self.assertIn('cookies', params)
self.assertIn('lang', params['cookies'])
self.assertIn('en', params['cookies']['lang'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, twitter.response, None) self.assertRaises(AttributeError, twitter.response, None)
self.assertRaises(AttributeError, twitter.response, []) self.assertRaises(AttributeError, twitter.response, [])

View File

@ -11,14 +11,14 @@ class TestWikidataEngine(SearxTestCase):
def test_request(self): def test_request(self):
query = 'test_query' query = 'test_query'
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['language'] = 'all' dicto['language'] = 'en-US'
params = wikidata.request(query, dicto) params = wikidata.request(query, dicto)
self.assertIn('url', params) self.assertIn('url', params)
self.assertIn(query, params['url']) self.assertIn(query, params['url'])
self.assertIn('wikidata.org', params['url']) self.assertIn('wikidata.org', params['url'])
self.assertIn('en', params['url']) self.assertIn('en', params['url'])
dicto['language'] = 'es_ES' dicto['language'] = 'es-ES'
params = wikidata.request(query, dicto) params = wikidata.request(query, dicto)
self.assertIn(query, params['url']) self.assertIn(query, params['url'])
self.assertIn('es', params['url']) self.assertIn('es', params['url'])
@ -30,7 +30,7 @@ class TestWikidataEngine(SearxTestCase):
self.assertRaises(AttributeError, wikidata.response, '') self.assertRaises(AttributeError, wikidata.response, '')
self.assertRaises(AttributeError, wikidata.response, '[]') self.assertRaises(AttributeError, wikidata.response, '[]')
response = mock.Mock(text='<html></html>', search_params={"language": "all"}) response = mock.Mock(text='<html></html>', search_params={"language": "en"})
self.assertEqual(wikidata.response(response), []) self.assertEqual(wikidata.response(response), [])
def test_getDetail(self): def test_getDetail(self):

View File

@ -25,10 +25,6 @@ class TestWikipediaEngine(SearxTestCase):
self.assertIn('Test_Query', params['url']) self.assertIn('Test_Query', params['url'])
self.assertNotIn('test_query', params['url']) self.assertNotIn('test_query', params['url'])
dicto['language'] = 'all'
params = wikipedia.request(query, dicto)
self.assertIn('en', params['url'])
dicto['language'] = 'xx' dicto['language'] = 'xx'
params = wikipedia.request(query, dicto) params = wikipedia.request(query, dicto)
self.assertIn('en', params['url']) self.assertIn('en', params['url'])

View File

@ -17,11 +17,6 @@ class TestYacyEngine(SearxTestCase):
self.assertIn('localhost', params['url']) self.assertIn('localhost', params['url'])
self.assertIn('fr', params['url']) self.assertIn('fr', params['url'])
dicto['language'] = 'all'
params = yacy.request(query, dicto)
self.assertIn('url', params)
self.assertNotIn('lr=lang_', params['url'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, yacy.response, None) self.assertRaises(AttributeError, yacy.response, None)
self.assertRaises(AttributeError, yacy.response, []) self.assertRaises(AttributeError, yacy.response, [])

View File

@ -39,13 +39,6 @@ class TestYahooEngine(SearxTestCase):
self.assertIn('sB', params['cookies']) self.assertIn('sB', params['cookies'])
self.assertIn('fr', params['cookies']['sB']) self.assertIn('fr', params['cookies']['sB'])
dicto['language'] = 'all'
params = yahoo.request(query, dicto)
self.assertIn('cookies', params)
self.assertIn('sB', params['cookies'])
self.assertIn('en', params['cookies']['sB'])
self.assertIn('en', params['url'])
def test_no_url_in_request_year_time_range(self): def test_no_url_in_request_year_time_range(self):
dicto = defaultdict(dict) dicto = defaultdict(dict)
query = 'test_query' query = 'test_query'

View File

@ -22,13 +22,6 @@ class TestYahooNewsEngine(SearxTestCase):
self.assertIn('sB', params['cookies']) self.assertIn('sB', params['cookies'])
self.assertIn('fr', params['cookies']['sB']) self.assertIn('fr', params['cookies']['sB'])
dicto['language'] = 'all'
params = yahoo_news.request(query, dicto)
self.assertIn('cookies', params)
self.assertIn('sB', params['cookies'])
self.assertIn('en', params['cookies']['sB'])
self.assertIn('en', params['url'])
def test_sanitize_url(self): def test_sanitize_url(self):
url = "test.url" url = "test.url"
self.assertEqual(url, yahoo_news.sanitize_url(url)) self.assertEqual(url, yahoo_news.sanitize_url(url))

View File

@ -90,25 +90,30 @@ class TestSettings(SearxTestCase):
# search language settings # search language settings
def test_lang_setting_valid_choice(self): def test_lang_setting_valid_choice(self):
setting = SearchLanguageSetting('all', choices=['all', 'de', 'en']) setting = SearchLanguageSetting('en', choices=['de', 'en'])
setting.parse('de') setting.parse('de')
self.assertEquals(setting.get_value(), 'de') self.assertEquals(setting.get_value(), 'de')
def test_lang_setting_invalid_choice(self): def test_lang_setting_invalid_choice(self):
setting = SearchLanguageSetting('all', choices=['all', 'de', 'en']) setting = SearchLanguageSetting('en', choices=['de', 'en'])
setting.parse('xx') setting.parse('xx')
self.assertEquals(setting.get_value(), 'all') self.assertEquals(setting.get_value(), 'en')
def test_lang_setting_old_cookie_choice(self): def test_lang_setting_old_cookie_choice(self):
setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES']) setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])
setting.parse('es_XA') setting.parse('es_XA')
self.assertEquals(setting.get_value(), 'es') self.assertEquals(setting.get_value(), 'es')
def test_lang_setting_old_cookie_format(self): def test_lang_setting_old_cookie_format(self):
setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES']) setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])
setting.parse('es_ES') setting.parse('es_ES')
self.assertEquals(setting.get_value(), 'es-ES') self.assertEquals(setting.get_value(), 'es-ES')
def test_lang_setting_old_default(self):
setting = SearchLanguageSetting('en', choices=['en', 'es', 'de'])
setting.parse('all')
self.assertEquals(setting.get_value(), 'en')
# plugins settings # plugins settings
def test_plugins_setting_all_default_enabled(self): def test_plugins_setting_all_default_enabled(self):
plugin1 = PluginStub('plugin1', True) plugin1 = PluginStub('plugin1', True)

64
tests/unit/test_query.py Normal file
View File

@ -0,0 +1,64 @@
from searx.query import RawTextQuery
from searx.testing import SearxTestCase
class TestQuery(SearxTestCase):
def test_simple_query(self):
query_text = 'the query'
query = RawTextQuery(query_text, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), query_text)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)
def test_language_code(self):
language = 'es-ES'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 3)
self.assertEquals(len(query.languages), 1)
self.assertIn(language, query.languages)
self.assertFalse(query.specific)
def test_language_name(self):
language = 'english'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 3)
self.assertIn('en', query.languages)
self.assertFalse(query.specific)
def test_unlisted_language_code(self):
language = 'all'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 3)
self.assertIn('all', query.languages)
self.assertFalse(query.specific)
def test_invalid_language_code(self):
language = 'not_a_language'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)