mirror of https://github.com/Kkevsterrr/geneva
264 lines
10 KiB
Python
264 lines
10 KiB
Python
|
"""
|
||
|
HTTP Plugin driver
|
||
|
|
||
|
Overrides the default evaluator plugin handling so we can negotiate a clear port
|
||
|
and track jailed sites to avoid residual censorship.
|
||
|
"""
|
||
|
|
||
|
import argparse
|
||
|
import calendar
|
||
|
import copy
|
||
|
import logging
|
||
|
import os
|
||
|
import random
|
||
|
import socket
|
||
|
import subprocess as sp
|
||
|
import sys
|
||
|
import tempfile
|
||
|
import time
|
||
|
import traceback
|
||
|
import urllib.request
|
||
|
|
||
|
import requests
|
||
|
|
||
|
socket.setdefaulttimeout(1)
|
||
|
|
||
|
import engine
|
||
|
import external_sites
|
||
|
import actions.utils
|
||
|
|
||
|
from plugins.plugin import Plugin
|
||
|
|
||
|
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||
|
PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH))
|
||
|
TEST_SITES = copy.deepcopy(external_sites.EXTERNAL_SITES)
|
||
|
JAIL_TRACKER = {}
|
||
|
for site in TEST_SITES:
|
||
|
JAIL_TRACKER[site] = 0
|
||
|
random.shuffle(TEST_SITES)
|
||
|
|
||
|
GOOD_WORD = "testing"
|
||
|
BAD_WORD = "ultrasurf"
|
||
|
JAIL_TIME = 95
|
||
|
|
||
|
|
||
|
class HTTPPluginRunner(Plugin):
|
||
|
"""
|
||
|
Defines the HTTP plugin runner.
|
||
|
"""
|
||
|
name = "http"
|
||
|
def __init__(self, args):
|
||
|
"""
|
||
|
Marks this plugin as enabled
|
||
|
"""
|
||
|
self.enabled = True
|
||
|
|
||
|
def negotiate_clear_port(self, args, evaluator, environment, logger):
|
||
|
"""
|
||
|
Since residual censorship might be affecting our IP/port combo
|
||
|
that was randomly chosen, this method is to find a port on which
|
||
|
no residual censorship is present. This is done simply by picking a
|
||
|
port, running the server, running curl to confirm it's accessible,
|
||
|
and then returning that port.
|
||
|
"""
|
||
|
while True:
|
||
|
# Pick a random port if port negotiation is enabled
|
||
|
if args["disable_port_negotiation"] and not args.get("censor"):
|
||
|
port = 80
|
||
|
else:
|
||
|
port = random.randint(1025, 65000)
|
||
|
port_bound = True
|
||
|
while port_bound:
|
||
|
try:
|
||
|
with socket.socket() as sock:
|
||
|
sock.bind(('', port))
|
||
|
# if no error thrown, port was unbound and thus we can use it
|
||
|
port_bound = False
|
||
|
except OSError:
|
||
|
# port was bound already, generate new port and try again
|
||
|
port = random.randint(1025, 65000)
|
||
|
|
||
|
# Store that port in the server args
|
||
|
evaluator.server_args.update({"port": port})
|
||
|
# Disable the engine so the strategy under test does not interfere with the
|
||
|
# residual censorship check
|
||
|
evaluator.server_args.update({"no_engine": True})
|
||
|
|
||
|
# Start the server on our chosen port
|
||
|
try:
|
||
|
http_server = evaluator.start_server(evaluator.server_args, environment, logger)
|
||
|
except:
|
||
|
logger.exception("Failed to start server; choosing a new port.")
|
||
|
continue
|
||
|
|
||
|
if args["disable_port_negotiation"] or args.get("censor"):
|
||
|
return http_server, port
|
||
|
|
||
|
# Test for residual censorship
|
||
|
dest = "%s:%d" % (evaluator.get_ip(), port)
|
||
|
logger.debug("Checking for residual censorship at %s" % dest)
|
||
|
command = "curl -s %s -m 5 -v" % dest
|
||
|
stdout, stderr = evaluator.remote_exec_cmd(environment["remote"], command, logger, timeout=7)
|
||
|
for line in stderr:
|
||
|
if "Connection reset by peer" in line:
|
||
|
logger.info("Residual censorship detected on %s." % dest)
|
||
|
evaluator.stop_server(environment, http_server)
|
||
|
time.sleep(1)
|
||
|
continue
|
||
|
elif "timed out" in line:
|
||
|
logger.debug("Connection timed out on %s" % dest)
|
||
|
evaluator.stop_server(environment, http_server)
|
||
|
raise actions.utils.SkipStrategyException("Strategy broke TCP connection", -400)
|
||
|
break
|
||
|
return http_server, port
|
||
|
|
||
|
def start(self, args, evaluator, environment, ind, logger):
|
||
|
"""
|
||
|
Runs the plugins
|
||
|
"""
|
||
|
if args["use_external_sites"]:
|
||
|
args.update({"external_server" : True})
|
||
|
|
||
|
forwarder = {}
|
||
|
if evaluator.act_as_middlebox:
|
||
|
forwarder["sender_ip"] = args.get("sender_ip")
|
||
|
forwarder["forward_ip"] = args.get("forward_ip")
|
||
|
forwarder["routing_ip"] = args.get("routing_ip")
|
||
|
|
||
|
port = args.get("port", 80)
|
||
|
tmp_dir = None
|
||
|
# If we're given a server to start, start it now, but if we're a middlebox, don't run a server
|
||
|
if evaluator.server_cls and not args.get("external_server") and not evaluator.act_as_middlebox:
|
||
|
server, port = self.negotiate_clear_port(args, evaluator, environment, logger)
|
||
|
|
||
|
# Update the port with the given or negotiated port
|
||
|
evaluator.client_args.update({"port": port})
|
||
|
site_to_test = evaluator.client_args.get("server", "")
|
||
|
|
||
|
output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory"))
|
||
|
|
||
|
with engine.Engine(port, args.get("strategy", ""), server_side=args["server_side"], environment_id=environment["id"], output_directory=output_path, log_level=args.get("log", "debug"), enabled=args["server_side"], forwarder=forwarder) as eng:
|
||
|
with TestServer(site_to_test, evaluator, environment, logger) as site_to_test:
|
||
|
evaluator.client_args.update({"server" : site_to_test})
|
||
|
fitness = evaluator.run_client(evaluator.client_args, environment, logger)
|
||
|
|
||
|
evaluator.read_fitness(ind)
|
||
|
|
||
|
# If the engine ran on the server side, ask that it punish fitness
|
||
|
if args["server_side"]:
|
||
|
ind.fitness = actions.utils.punish_fitness(fitness, logger, eng)
|
||
|
actions.utils.write_fitness(ind.fitness, output_path, environment["id"])
|
||
|
|
||
|
if evaluator.server_cls and not evaluator.args.get("external_server") and not evaluator.act_as_middlebox:
|
||
|
evaluator.stop_server(environment, server)
|
||
|
|
||
|
return ind.environment_id, ind.fitness
|
||
|
|
||
|
@staticmethod
|
||
|
def get_args(command):
|
||
|
"""
|
||
|
Defines required global args for all plugins
|
||
|
"""
|
||
|
parser = argparse.ArgumentParser(description='HTTP plugin runner', allow_abbrev=False)
|
||
|
parser.add_argument('--disable-port-negotiation', action='store_true', help="disables port negotiation between remote client and local server")
|
||
|
parser.add_argument('--use-external-sites', action='store_true', help="draw from the pool of external servers (defined in external_sites.py) for testing.")
|
||
|
parser.add_argument('--environment-id', action='store', help="ID of the current environment")
|
||
|
parser.add_argument('--output-directory', action='store', help="Where to output results")
|
||
|
parser.add_argument('--port', action='store', type=int, default=80, help='port to use')
|
||
|
args, _ = parser.parse_known_args(command)
|
||
|
return vars(args)
|
||
|
|
||
|
|
||
|
def check_censorship(site, evaluator, environment, logger):
|
||
|
"""
|
||
|
Make a request to the given site to test if it is censored. Used to test
|
||
|
a site for residual censorship before using it.
|
||
|
"""
|
||
|
command = "curl -s %s -m 5" % site
|
||
|
if environment.get("remote"):
|
||
|
stdout, stderr = evaluator.remote_exec_cmd(environment["remote"], command, logger, timeout=5)
|
||
|
for line in stderr:
|
||
|
if "Connection reset by peer" in line:
|
||
|
logger.info("Residual censorship detected on %s." % site)
|
||
|
return False
|
||
|
return True
|
||
|
try:
|
||
|
requests.get(site, allow_redirects=False, timeout=3)
|
||
|
return True
|
||
|
except (requests.exceptions.ConnectionError,
|
||
|
ConnectionResetError,
|
||
|
urllib.error.URLError,
|
||
|
requests.exceptions.Timeout,
|
||
|
Exception) as e:
|
||
|
logger.error("Could not reach site %s" % site)
|
||
|
return False
|
||
|
|
||
|
|
||
|
class TestServer():
|
||
|
"""
|
||
|
Context manager to retrieve a test server from the external server pool.
|
||
|
"""
|
||
|
def __init__(self, requested_site, evaluator, environment, logger):
|
||
|
self.requested_site = requested_site
|
||
|
self.logger = logger
|
||
|
self.evaluator = evaluator
|
||
|
self.environment = environment
|
||
|
self.site_to_test = None
|
||
|
|
||
|
def __enter__(self):
|
||
|
"""
|
||
|
Reserves a site for testing for this worker.
|
||
|
"""
|
||
|
if self.requested_site:
|
||
|
return self.requested_site
|
||
|
|
||
|
while True:
|
||
|
site_to_test = TEST_SITES.pop(0)
|
||
|
current_seconds = calendar.timegm(time.gmtime())
|
||
|
|
||
|
# Check if our current time is at least JAIL_TIME away from the last time
|
||
|
# we tried to use this site.
|
||
|
if (current_seconds - JAIL_TRACKER[site_to_test]) > JAIL_TIME:
|
||
|
site_good = False
|
||
|
self.logger.debug("Checking %s for censorship." % site_to_test)
|
||
|
site_good = check_censorship(site_to_test, self.evaluator, self.environment, self.logger)
|
||
|
|
||
|
if site_good:
|
||
|
self.logger.debug("Using site %s for testing." % site_to_test)
|
||
|
break
|
||
|
else:
|
||
|
self.logger.debug("Residual censorship detected for %s" % site_to_test)
|
||
|
|
||
|
# If we didn't break, put the site back at the end of the list
|
||
|
TEST_SITES.append(site_to_test)
|
||
|
if self.logger:
|
||
|
self.logger.debug("%s is not yet available to test - only %d seconds have \
|
||
|
transpired since last test - waiting 5 seconds." %
|
||
|
(site_to_test, current_seconds - JAIL_TRACKER[site_to_test]))
|
||
|
time.sleep(0.1)
|
||
|
|
||
|
# Store the site we're testing to re-add it to the pool on exit
|
||
|
self.site_to_test = site_to_test
|
||
|
return site_to_test
|
||
|
|
||
|
def __exit__(self, exc_type, exc_value, trace):
|
||
|
"""
|
||
|
Cleans up and returns the site in testing to the test pool.
|
||
|
"""
|
||
|
if self.site_to_test:
|
||
|
self.logger.debug("Returning %s to pool of sites." % self.site_to_test)
|
||
|
TEST_SITES.append(self.site_to_test)
|
||
|
JAIL_TRACKER[self.site_to_test] = calendar.timegm(time.gmtime())
|
||
|
|
||
|
# Pass through exceptions
|
||
|
if exc_type is not None:
|
||
|
traceback.print_exception(exc_type, exc_value, trace)
|
||
|
return False
|
||
|
return True
|
||
|
|
||
|
|
||
|
# Note that this code is not for debugging and cannot be removed -
|
||
|
# this is how the evaluator runs the client.
|
||
|
if __name__ == "__main__":
|
||
|
main(vars(get_args()))
|