From 9b9f097adbf39d7908931203e9d8966748900bde Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 14 Sep 2014 11:09:44 +0200 Subject: [PATCH] Implementing https rewrite support #71 * parsing XML-Files which contain target, exclusions and rules * convert regex if required (is a little hack, probably does not work for all rules) * check if target rule apply for http url, and use the rules to rewrite it * add pice of code, to check if domain name has not changed during rewrite (should be rewritten, using publicsuffix instead of little hack) --- searx/__init__.py | 8 +++ searx/https_rewrite.py | 141 ++++++++++++++++++++++++++++++++++++++--- searx/webapp.py | 52 +++++++++++++-- 3 files changed, 187 insertions(+), 14 deletions(-) diff --git a/searx/__init__.py b/searx/__init__.py index 375a5414..0acc6144 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -1,5 +1,6 @@ from os import environ from os.path import realpath, dirname, join, abspath +from searx.https_rewrite import load_https_rules try: from yaml import load except: @@ -15,6 +16,13 @@ if 'SEARX_SETTINGS_PATH' in environ: else: settings_path = join(searx_dir, 'settings.yml') +if 'SEARX_HTTPS_REWRITE_PATH' in environ: + https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH'] +else: + https_rewrite_path = join(searx_dir, 'https_rules') with open(settings_path) as settings_yaml: settings = load(settings_yaml) + +# loade https rules +load_https_rules(https_rewrite_path) diff --git a/searx/https_rewrite.py b/searx/https_rewrite.py index 44ada945..814eda2d 100644 --- a/searx/https_rewrite.py +++ b/searx/https_rewrite.py @@ -1,14 +1,139 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2013- by Adam Tauber, +''' + import re +from lxml import etree +from os import listdir +from os.path import isfile, join + # https://gitweb.torproject.org/\ # pde/https-everywhere.git/tree/4.0:/src/chrome/content/rules # HTTPS rewrite rules -https_rules = ( - # from - (re.compile(r'^http://(www\.|m\.|)?xkcd\.(?:com|org)/', re.I | re.U), - # to - r'https://\1xkcd.com/'), - (re.compile(r'^https?://(?:ssl)?imgs\.xkcd\.com/', re.I | re.U), - r'https://sslimgs.xkcd.com/'), -) +https_rules = [] + + +# load single ruleset from a xml file +def load_single_https_ruleset(filepath): + ruleset = () + + # init parser + parser = etree.XMLParser() + + # load and parse xml-file + try: + tree = etree.parse(filepath, parser) + except: + # TODO, error message + return () + + # get root node + root = tree.getroot() + + #print(etree.tostring(tree)) + + # check if root is a node with the name ruleset + # TODO improve parsing + if root.tag != 'ruleset': + return () + + # check if rule is deactivated by default + if root.attrib.get('default_off'): + return () + + # check if rule does only work for specific platforms + if root.attrib.get('platform'): + return () + + hosts = [] + rules = [] + exclusions = [] + + # parse childs from ruleset + for ruleset in root: + # this child define a target + if ruleset.tag == 'target': + # check if required tags available + if not ruleset.attrib.get('host'): + continue + + # convert host-rule to valid regex + host = ruleset.attrib.get('host').replace('.', '\.').replace('*', '.*') + + # append to host list + hosts.append(host) + + # this child define a rule + elif ruleset.tag == 'rule': + # check if required tags available + if not ruleset.attrib.get('from')\ + or not ruleset.attrib.get('to'): + continue + + # TODO hack, which convert a javascript regex group into a valid python regex group + rule_from = ruleset.attrib.get('from').replace('$', '\\') + rule_to = ruleset.attrib.get('to').replace('$', '\\') + + # TODO, not working yet because of the hack above, currently doing that in webapp.py + #rule_from_rgx = re.compile(rule_from, re.I) + + # append rule + rules.append((rule_from, rule_to)) + + # this child define an exclusion + elif ruleset.tag == 'exclusion': + # check if required tags available + if not ruleset.attrib.get('pattern'): + continue + + exclusion_rgx = re.compile(ruleset.attrib.get('pattern')) + + # append exclusion + exclusions.append(exclusion_rgx) + + # convert list of possible hosts to a simple regex + # TODO compress regex to improve performance + try: + target_hosts = re.compile('^(' + '|'.join(hosts) + ')', re.I | re.U) + except: + return () + + # return ruleset + return (target_hosts, rules, exclusions) + + +# load all https rewrite rules +def load_https_rules(rules_path): + # add / to path if not set yet + if rules_path[-1:] != '/': + rules_path += '/' + + # search all xml files which are stored in the https rule directory + xml_files = [ join(rules_path,f) for f in listdir(rules_path) if isfile(join(rules_path,f)) and f[-4:] == '.xml' ] + + # load xml-files + for ruleset_file in xml_files: + # calculate rewrite-rules + ruleset = load_single_https_ruleset(ruleset_file) + + # skip if no ruleset returned + if not ruleset: + continue + + # append ruleset + https_rules.append(ruleset) diff --git a/searx/webapp.py b/searx/webapp.py index 2bf3afaf..7952415a 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -49,6 +49,9 @@ from searx.languages import language_codes from searx.search import Search from searx.autocomplete import backends as autocomplete_backends +from urlparse import urlparse +import re + static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -197,16 +200,53 @@ def index(): if not search.paging and engines[result['engine']].paging: search.paging = True + # check if HTTPS rewrite is required if settings['server']['https_rewrite']\ and result['parsed_url'].scheme == 'http': - for http_regex, https_url in https_rules: - if http_regex.match(result['url']): - result['url'] = http_regex.sub(https_url, result['url']) - # TODO result['parsed_url'].scheme - break + skip_https_rewrite = False + + # check if HTTPS rewrite is possible + for target, rules, exclusions in https_rules: + + # check if target regex match with url + if target.match(result['url']): + # process exclusions + for exclusion in exclusions: + # check if exclusion match with url + if exclusion.match(result['url']): + skip_https_rewrite = True + break + + # skip https rewrite if required + if skip_https_rewrite: + break + + # process rules + for rule in rules: + # TODO, precompile rule + p = re.compile(rule[0]) + # rewrite url if possible + new_result_url = p.sub(rule[1], result['url']) + + # parse new url + new_parsed_url = urlparse(new_result_url) + + # continiue if nothing was rewritten + if result['url'] == new_result_url: + continue + + # get domainname from result + # TODO, does only work correct with TLD's like asdf.com, not for asdf.com.de + # TODO, using publicsuffix instead of this rewrite rule + old_result_domainname = '.'.join(result['parsed_url'].hostname.split('.')[-2:]) + new_result_domainname = '.'.join(new_parsed_url.hostname.split('.')[-2:]) + + # check if rewritten hostname is the same, to protect against wrong or malicious rewrite rules + if old_result_domainname == new_result_domainname: + # set new url + result['url'] = new_result_url - # HTTPS rewrite if search.request_data.get('format', 'html') == 'html': if 'content' in result: result['content'] = highlight_content(result['content'],