From de6823ba7723582054d2047083262cabffa85f36 Mon Sep 17 00:00:00 2001 From: Kkevsterrr Date: Fri, 7 Aug 2020 15:13:13 -0400 Subject: [PATCH] Added ESNI plugin --- actions/tamper.py | 2 +- layers/tcp_layer.py | 3 +- plugins/esni/client.py | 81 +++++++++++++++++++++++++ plugins/esni/plugin.py | 102 +++++++++++++++++++++++++++++++ plugins/esni/server.py | 134 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 plugins/esni/client.py create mode 100644 plugins/esni/plugin.py create mode 100644 plugins/esni/server.py diff --git a/actions/tamper.py b/actions/tamper.py index e1882b2..51242ac 100644 --- a/actions/tamper.py +++ b/actions/tamper.py @@ -21,7 +21,7 @@ import random SUPPORTED_PRIMITIVES = ["corrupt", "replace", "add", "compress"] # Tamper primitives we can mutate to by default -ACTIVATED_PRIMITIVES = ["replace", "corrupt", "add"] +ACTIVATED_PRIMITIVES = ["replace", "corrupt"] class TamperAction(Action): diff --git a/layers/tcp_layer.py b/layers/tcp_layer.py index 0f50797..668669f 100644 --- a/layers/tcp_layer.py +++ b/layers/tcp_layer.py @@ -1,3 +1,4 @@ +import os import random import binascii from layers.layer import Layer @@ -282,4 +283,4 @@ class TCPLayer(Layer): elif options_int in self.scapy_options: return (self.scapy_options[options_int], value) else: - return (options_int, value) \ No newline at end of file + return (options_int, value) diff --git a/plugins/esni/client.py b/plugins/esni/client.py new file mode 100644 index 0000000..f704224 --- /dev/null +++ b/plugins/esni/client.py @@ -0,0 +1,81 @@ +""" +Client + +Run by the evaluator, sends a TLS Client Hello with the ESNI extension, followed by two test packets. +""" + +import argparse +import binascii as bi +import os +import socket +import time + +socket.setdefaulttimeout(1) + +from plugins.plugin_client import ClientPlugin + + +class ESNIClient(ClientPlugin): + """ + Defines the ESNI client. + """ + name = "esni" + + def __init__(self, args): + """ + Initializes the esni client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='ESNI Client') + + parser.add_argument('--server', action='store', help="server to connect to") + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden GET request to the server. + """ + fitness = 0 + port = int(args["port"]) + server = args["server"] + # Client Hello with the ESNI extension + msg = b'16030103ae010003aa0303d992f9c22fbe7a7cdbc9619924bd9cc13c057f5f3da1829426cb0944292705152033c5be80af6de7633e07680125e27e3f7b80ff5e9b3cbe5278434c90b9e0e5fa0024130113031302c02bc02fcca9cca8c02cc030c00ac009c013c014009c009d002f0035000a0100033d00170000ff01000100000a000e000c001d00170018001901000101000b000201000010000e000c02683208687474702f312e310005000501000000000033006b0069001d002019570ada256d971048b34d3e9ff5607588bf10cfb6c064fc45a0fc401d9a7c470017004104ea047fd2e0fc3314de4bf03ee6205134f0d15c07f62b77625a95dc194ce8fb88cc16e53c8b400ba463915b87480b247851c095abdb0d3d5d5b14dd77dcd73750002b00050403040303000d0018001604030503060308040805080604010501060102030201002d00020101ffce016e1301001d00203652aaf122dc47dcf9fa8c37377476d050e54119adfb518f7aabd842ac97d23b00205a30e70593f57708370310ecf7054e488a62eb11e01fd059851c442d453d15c5012441910eec152c4df5ff28bf5cddb1a2e54e8595197e3dc36325145ad50a7842eb3860c8fc6ac5c1794017101365c6122abb3b81f31f5f4204eebb244252d22600734424d875948657b892d3aab3310491aff3b5126f1186bd9c321fb446cf2a41985dd206364ea28c3f8aafeafc62e039f157c3f2703a35448d2d16dcf2d5055ce58c024a5b4eb780fc5128af4ba4e90d6eef1b3cf30a5b2000448d65d6af4fffabeb91e1ed2093fdcc6ffd87ceb94429864ddb657e6316654631193fd25840e51645e1708d351140dd6eeefb80ddbaebb250b2975a1d5f291d99f89de4553d083f1b9820a3ee6976357cff433b7eb77febb3eb0db012154154d3e19b4409f8afa11aa1baeb0b7663d97f0caca2b11ed971fc574588e76a37aa4259593fe8e07fbbca27fa001c00024001002900eb00c600c07f87fafe9de4168227aeec4540f1aaeae43ff61a353f5480420ac3c33f90003fe6f501080bf04f22576a0cc1db8dc83d37b25859a81ce0277364a1794cde1c60f3b94175477beff56db7f9e2b83b31383b7d8b5da20834fb0a63d7ba2e42ad3dfa21666ed8621f34273ac5c273d7f492750e3df3bae36e398ddf83d4a7c36f639087f14eb1f7bfb2c7c0c736d69bcdbf21158c07b7088b95e5bcd08138d6b511f6492d7d93bb3729641519097b970cfeffa5882c67111dcf5d7966a1c58b4edb6e8c905a002120e47ccba37d89e4c1d979c6ef954d1cd946eff0d3119aa2b4d6411138aec74579' + try: + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.settimeout(5) + client.connect((server, port)) + client.sendall(bi.unhexlify(msg)) + time.sleep(2) + client.sendall(b"test packet") + time.sleep(2) + client.sendall(b"test packet 2") + server_data = client.recv(1024) + logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore')) + fitness += 100 + client.close() + except socket.timeout: + # Happens on connect, not sendall + logger.debug("Client: Timeout") + fitness -= 110 + except socket.error as exc: + fitness -= 100 + logger.exception("Socket error caught in client esni test.") + except Exception: + logger.exception("Exception caught in client esni test.") + fitness = -120 + finally: + logger.debug("Client finished esni test.") + return fitness * 4 diff --git a/plugins/esni/plugin.py b/plugins/esni/plugin.py new file mode 100644 index 0000000..e5e68a8 --- /dev/null +++ b/plugins/esni/plugin.py @@ -0,0 +1,102 @@ +""" +ESNI Plugin driver + +Overrides the default evaluator plugin handling so we can check if the server timed out on recv. +""" + +import argparse +import calendar +import copy +import logging +import os +import random +import socket +import sys +import tempfile +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import actions.utils + +from plugins.plugin import Plugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH)) + + +class ESNIPluginRunner(Plugin): + """ + Defines the ESNI plugin runner. + """ + name = "esni" + + def __init__(self, args): + """ + Marks this plugin as enabled + """ + self.enabled = True + + def start(self, args, evaluator, environment, ind, logger): + """ + Runs the plugins + """ + # Start the server + port = random.randint(10000, 65000) + evaluator.client_args.update({"port": port}) + evaluator.server_args.update({"port": port}) + + # If we're given a server to start, start it now + if evaluator.server_cls and not args.get("external_server"): + # If a test using TCP has been requested, switch the server to that mode + server = evaluator.start_server(evaluator.server_args, environment, logger) + evaluator.client_args.update({"server": evaluator.args["server"]}) + + fitness = evaluator.run_client(evaluator.client_args, environment, logger) + + if evaluator.server_cls and not evaluator.args["external_server"]: + evaluator.stop_server(environment, server) + + evaluator.read_fitness(ind) + + # If the engine ran on the server side, ask that it punish fitness + if evaluator.args["server_side"]: + ind.fitness = server.punish_fitness(ind.fitness, logger) + output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory")) + fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness" + with open(fitpath, "w") as fitfile: + fitfile.write(str(ind.fitness)) + + if evaluator.server_cls and not evaluator.args["external_server"]: + logger.debug("CHECKING FOR SERVER TIMEOUT") + output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory")) + timeout_flag = os.path.join(output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".timeout" + fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness" + if os.path.exists(timeout_flag): + logger.debug("Server timeout detected") + ind.fitness = -360 + with open(fitpath, "w") as fitfile: + fitfile.write(str(ind.fitness)) + + evaluator.read_fitness(ind) + + # Log the fitness + #logger.info("[%s] Fitness %s: %s" % (ind.environment_id, str(ind.fitness), str(ind))) + + return ind.environment_id, ind.fitness + + @staticmethod + def get_args(command): + """ + Defines required global args for this plugin + """ + parser = argparse.ArgumentParser(description='ESNI plugin runner', allow_abbrev=False) + parser.add_argument('--environment-id', action='store', help="ID of the current environment") + parser.add_argument('--output-directory', action='store', help="Where to output results") + parser.add_argument('--port', action='store', type=int, help='port to use') + args, _ = parser.parse_known_args(command) + return vars(args) diff --git a/plugins/esni/server.py b/plugins/esni/server.py new file mode 100644 index 0000000..7c398cf --- /dev/null +++ b/plugins/esni/server.py @@ -0,0 +1,134 @@ +""" +ESNI Test Server + +Starts a simple TCP server, recvs data until it gets the bytes it expects from the client. +""" +import argparse +import binascii as bi +import os +import socket + +from plugins.plugin_server import ServerPlugin +import actions.utils + + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH)) + + +class ESNIServer(ServerPlugin): + """ + Defines the ESNI client. + """ + name = "esni" + def __init__(self, args): + """ + Initializes the ESNI client. + """ + ServerPlugin.__init__(self) + + @staticmethod + def get_args(command): + """ + Defines arguments for this plugin + """ + super_args = ServerPlugin.get_args(command) + + parser = argparse.ArgumentParser(description='ESNI Test Server') + + args, _ = parser.parse_known_args(command) + args = vars(args) + super_args.update(args) + return super_args + + def run(self, args, logger): + """ + Initializes the ESNI server. + """ + logger.debug("ESNI test server initializing") + try: + port = int(args["port"]) + control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Allow socket re-use + control_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_address = ('0.0.0.0', port) + logger.debug("Binding to server address 0.0.0.0:%d" % port) + control_socket.bind(server_address) + control_socket.settimeout(10) + control_socket.listen(1) + except: + logger.exception("Caught exception in esni run") + return + needed = bi.unhexlify(b'16030103ae010003aa0303d992f9c22fbe7a7cdbc9619924bd9cc13c057f5f3da1829426cb0944292705152033c5be80af6de7633e07680125e27e3f7b80ff5e9b3cbe5278434c90b9e0e5fa0024130113031302c02bc02fcca9cca8c02cc030c00ac009c013c014009c009d002f0035000a0100033d00170000ff01000100000a000e000c001d00170018001901000101000b000201000010000e000c02683208687474702f312e310005000501000000000033006b0069001d002019570ada256d971048b34d3e9ff5607588bf10cfb6c064fc45a0fc401d9a7c470017004104ea047fd2e0fc3314de4bf03ee6205134f0d15c07f62b77625a95dc194ce8fb88cc16e53c8b400ba463915b87480b247851c095abdb0d3d5d5b14dd77dcd73750002b00050403040303000d0018001604030503060308040805080604010501060102030201002d00020101ffce016e1301001d00203652aaf122dc47dcf9fa8c37377476d050e54119adfb518f7aabd842ac97d23b00205a30e70593f57708370310ecf7054e488a62eb11e01fd059851c442d453d15c5012441910eec152c4df5ff28bf5cddb1a2e54e8595197e3dc36325145ad50a7842eb3860c8fc6ac5c1794017101365c6122abb3b81f31f5f4204eebb244252d22600734424d875948657b892d3aab3310491aff3b5126f1186bd9c321fb446cf2a41985dd206364ea28c3f8aafeafc62e039f157c3f2703a35448d2d16dcf2d5055ce58c024a5b4eb780fc5128af4ba4e90d6eef1b3cf30a5b2000448d65d6af4fffabeb91e1ed2093fdcc6ffd87ceb94429864ddb657e6316654631193fd25840e51645e1708d351140dd6eeefb80ddbaebb250b2975a1d5f291d99f89de4553d083f1b9820a3ee6976357cff433b7eb77febb3eb0db012154154d3e19b4409f8afa11aa1baeb0b7663d97f0caca2b11ed971fc574588e76a37aa4259593fe8e07fbbca27fa001c00024001002900eb00c600c07f87fafe9de4168227aeec4540f1aaeae43ff61a353f5480420ac3c33f90003fe6f501080bf04f22576a0cc1db8dc83d37b25859a81ce0277364a1794cde1c60f3b94175477beff56db7f9e2b83b31383b7d8b5da20834fb0a63d7ba2e42ad3dfa21666ed8621f34273ac5c273d7f492750e3df3bae36e398ddf83d4a7c36f639087f14eb1f7bfb2c7c0c736d69bcdbf21158c07b7088b95e5bcd08138d6b511f6492d7d93bb3729641519097b970cfeffa5882c67111dcf5d7966a1c58b4edb6e8c905a002120e47ccba37d89e4c1d979c6ef954d1cd946eff0d3119aa2b4d6411138aec74579') + b'test packet' + b'test packet 2' + connection = None + try: + connection, client_address = self.get_request(control_socket) + if not connection: + logger.error("Failed to get connection") + return + data = b"" + while len(data) < len(needed): + logger.debug("Awaiting data") + d = connection.recv(256) + if not d: + break + logger.debug(b"Received: " + d) + logger.debug("Got: %d; Remaining: %d", len(data), len(needed)) + logger.debug(len(d)) + data += d + logger.debug("Got %d; Remaining: %d", len(data), len(needed)) + connection.sendall(b'ack') + assert data == needed, data + logger.debug("Successfully got all of the client's data") + + connection.close() + except socket.timeout as e: + # write to a flag file to pass back to the plugin that this strategy failed + logger.debug("Server: Connection timed out") + flagpath = os.path.join(PROJECT_ROOT, args["output_directory"], actions.utils.FLAGFOLDER, args["environment_id"]) + ".timeout" + with open(flagpath, "w") as fd: + fd.write("timeout caught") + fd.flush() + except AssertionError as exc: + logger.debug("Did not receive all data: probably a client timeout") + flagpath = os.path.join(PROJECT_ROOT, args["output_directory"], actions.utils.FLAGFOLDER, args["environment_id"]) + ".timeout" + with open(flagpath, "w") as fd: + fd.write("timeout caught") + fd.flush() + except socket.error as e: + if e.errno == 104: + logger.debug("Server: Connection RST.") + else: + logger.debug("Server: Client quit.") + except AssertionError: + logger.debug("Server: Got incorrect data. Client's packets getting dropped?") + flagpath = os.path.join(PROJECT_ROOT, args["output_directory"], actions.utils.FLAGFOLDER, args["environment_id"]) + ".timeout" + with open(flagpath, "w") as fd: + fd.write("packets dropped") + fd.flush() + + except Exception: + logger.exception("Failed during server communication.") + finally: + if connection: + connection.close() + logger.debug("Server exiting") + + def get_request(self, control_socket): + """ + Get a request from the socket. + """ + while True: + try: + sock, addr = control_socket.accept() + sock.settimeout(5) + return (sock, addr) + except socket.timeout: + pass + return (None, None) + + def stop(self): + """ + Stops this server. + """ + ServerPlugin.stop(self)