From 3eda6a1ca9c3d7b6e9b3bb543f54e3c5253aa3c1 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Mon, 2 Dec 2019 23:22:47 -0800 Subject: [PATCH 01/13] Initial Commit of Windows port --- actions/utils.py | 4 + engine.py | 292 ++++++++++++++++------------------------------- requirements.txt | 2 +- 3 files changed, 103 insertions(+), 195 deletions(-) diff --git a/actions/utils.py b/actions/utils.py index 5bad6ee..97585e5 100644 --- a/actions/utils.py +++ b/actions/utils.py @@ -221,6 +221,8 @@ def get_interface(): """ Chooses an interface on the machine to use for socket testing. """ + """ + TODO: FIX ifaces = netifaces.interfaces() for iface in ifaces: if "lo" in iface: @@ -229,3 +231,5 @@ def get_interface(): # Filter for IPv4 addresses if netifaces.AF_INET in info: return iface + """ + return "Wi-Fi" diff --git a/engine.py b/engine.py index 7f45e08..a8f13e1 100644 --- a/engine.py +++ b/engine.py @@ -4,7 +4,6 @@ Engine Given a strategy and a server port, the engine configures NFQueue so the strategy can run on the underlying connection. """ - import argparse import logging logging.getLogger("scapy.runtime").setLevel(logging.ERROR) @@ -14,11 +13,13 @@ import subprocess import threading import time -import netfilterqueue - from scapy.layers.inet import IP from scapy.utils import wrpcap from scapy.config import conf +from scapy.all import send, Raw + +import pydivert #TODO +from pydivert.consts import Direction socket.setdefaulttimeout(1) @@ -52,224 +53,131 @@ class Engine(): # Used for conditional context manager usage self.strategy = actions.utils.parse(string_strategy, self.logger) - # Setup variables used by the NFQueue system - self.out_nfqueue_started = False - self.in_nfqueue_started = False - self.running_nfqueue = False - self.out_nfqueue = None - self.in_nfqueue = None - self.out_nfqueue_socket = None - self.in_nfqueue_socket = None - self.out_nfqueue_thread = None - self.in_nfqueue_thread = None + + # Instantialize a PyDivert channel, which we will use to redirect packets + self.divert = None + self.divert_thread = None + self.divert_thread_started = False + self.censorship_detected = False # Specifically define an L3Socket to send our packets. This is an optimization # for scapy to send packets more quickly than using just send(), as under the hood # send() creates and then destroys a socket each time, imparting a large amount # of overhead. - self.socket = conf.L3socket(iface=actions.utils.get_interface()) + self.socket = conf.L3socket(iface=actions.utils.get_interface()) # TODO: FIX + + def initialize_divert(self): + """ + Initializes Divert such that all packets for the connection will come through us + """ + + self.logger.debug("Engine created with strategy %s (ID %s) to port %s", + str(self.strategy).strip(), self.environment_id, self.server_port) + + self.logger.debug("Initializing Divert") + + self.divert = pydivert.WinDivert("tcp.DstPort == %d || tcp.SrcPort == %d" % (int(self.server_port), int(self.server_port))) + self.divert.open() + self.divert_thread = threading.Thread(target=self.run_divert) + self.divert_thread.start() + + maxwait = 100 # 100 time steps of 0.01 seconds for a max wait of 10 seconds + i = 0 + # Give Divert time to startup, since it's running in background threads + # Block the main thread until this is done + while not self.divert_thread_started and i < maxwait: + time.sleep(0.1) + i += 1 + self.logger.debug("Divert Initialized after %d", int(i)) + + return + + def shutdown_divert(self): + """ + Closes the divert connection + """ + if self.divert: + self.divert.close() + self.divert = None + + return + + def run_divert(self): + """ + Runs actions on packets + """ + if self.divert: + self.divert_thread_started = True + + for packet in self.divert: + if packet.is_outbound: + # Send to outbound action tree, if any + self.handle_outbound_packet(packet) + + elif packet.is_inbound: + # Send to inbound action tree, if any + self.handle_inbound_packet(packet) + + return def __enter__(self): """ - Allows the engine to be used as a context manager; simply launches the - engine. + TODO """ - self.initialize_nfqueue() return self def __exit__(self, exc_type, exc_value, tb): """ - Allows the engine to be used as a context manager; simply stops the engine + TODO """ - self.shutdown_nfqueue() + return - def mysend(self, packet): + def mysend(self, packet, dir): """ Helper scapy sending method. Expects a Geneva Packet input. """ try: self.logger.debug("Sending packet %s", str(packet)) - self.socket.send(packet.packet) + + #Convert packet to pydivert + + #print(bytes(Raw(packet.packet))) + #print(packet.packet) + pack = bytes(packet.packet) + pack2 = bytearray(pack) + #print(pack2[0]) + #send(IP(packet.packet), iface="Wi-Fi") + #pack = bytearray(bytes(Raw(packet.packet)), "UTF-8") + #print(pack) + self.divert.send(pydivert.Packet(memoryview(pack2), (12, 0), dir), recalculate_checksum=False) # TODO: FIX + except Exception: self.logger.exception("Error in engine mysend.") - - def delayed_send(self, packet, delay): + + def handle_outbound_packet(self, divert_packet): """ - Method to be started by a thread to delay the sending of a packet without blocking the main thread. + Handles outbound packets by sending them the the strategy """ - self.logger.debug("Sleeping for %f seconds." % delay) - time.sleep(delay) - self.mysend(packet) - - def run_nfqueue(self, nfqueue, nfqueue_socket, direction): - """ - Handles running the outbound nfqueue socket with the socket timeout. - """ - try: - while self.running_nfqueue: - try: - if direction == "out": - self.out_nfqueue_started = True - else: - self.in_nfqueue_started = True - - nfqueue.run_socket(nfqueue_socket) - except socket.timeout: - pass - except Exception: - self.logger.exception("Exception out of run_nfqueue() (direction=%s)", direction) - - def configure_iptables(self, remove=False): - """ - Handles setting up ipables for this run - """ - self.logger.debug("Configuring iptables rules") - - port1, port2 = "dport", "sport" - - out_chain = "OUTPUT" - in_chain = "INPUT" - - # Switch whether the command should add or delete the rules - add_or_remove = "A" - if remove: - add_or_remove = "D" - cmds = [] - for proto in ["tcp", "udp"]: - cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 1" % - (add_or_remove, out_chain, proto, port1, self.server_port), - "iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 2" % - (add_or_remove, in_chain, proto, port2, self.server_port)] - - for cmd in cmds: - self.logger.debug(cmd) - # If we're logging at DEBUG mode, keep stderr/stdout piped to us - # Otherwise, pipe them both to DEVNULL - if actions.utils.get_console_log_level() == logging.DEBUG: - subprocess.check_call(cmd.split(), timeout=60) - else: - subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) - return cmds - - def initialize_nfqueue(self): - """ - Initializes the nfqueue for input and output forests. - """ - self.logger.debug("Engine created with strategy %s (ID %s) to port %s", - str(self.strategy).strip(), self.environment_id, self.server_port) - self.configure_iptables() - - self.out_nfqueue_started = False - self.in_nfqueue_started = False - self.running_nfqueue = True - # Create our NFQueues - self.out_nfqueue = netfilterqueue.NetfilterQueue() - self.in_nfqueue = netfilterqueue.NetfilterQueue() - # Bind them - self.out_nfqueue.bind(1, self.out_callback) - self.in_nfqueue.bind(2, self.in_callback) - # Create our nfqueue sockets to allow for non-blocking usage - self.out_nfqueue_socket = socket.fromfd(self.out_nfqueue.get_fd(), - socket.AF_UNIX, - socket.SOCK_STREAM) - self.in_nfqueue_socket = socket.fromfd(self.in_nfqueue.get_fd(), - socket.AF_UNIX, - socket.SOCK_STREAM) - # Create our handling threads for packets - self.out_nfqueue_thread = threading.Thread(target=self.run_nfqueue, - args=(self.out_nfqueue, self.out_nfqueue_socket, "out")) - - self.in_nfqueue_thread = threading.Thread(target=self.run_nfqueue, - args=(self.in_nfqueue, self.in_nfqueue_socket, "in")) - # Start each thread - self.in_nfqueue_thread.start() - self.out_nfqueue_thread.start() - - maxwait = 100 # 100 time steps of 0.01 seconds for a max wait of 10 seconds - i = 0 - # Give NFQueue time to startup, since it's running in background threads - # Block the main thread until this is done - while (not self.in_nfqueue_started or not self.out_nfqueue_started) and i < maxwait: - time.sleep(0.1) - i += 1 - self.logger.debug("NFQueue Initialized after %d", int(i)) - - def shutdown_nfqueue(self): - """ - Shutdown nfqueue. - """ - self.logger.debug("Shutting down NFQueue") - self.out_nfqueue_started = False - self.in_nfqueue_started = False - self.running_nfqueue = False - # Give the handlers two seconds to leave the callbacks before we forcibly unbind - # the queues. - time.sleep(2) - if self.in_nfqueue: - self.in_nfqueue.unbind() - if self.out_nfqueue: - self.out_nfqueue.unbind() - self.configure_iptables(remove=True) - - packets_path = os.path.join(BASEPATH, - self.output_directory, - "packets", - "original_%s.pcap" % self.environment_id) - - # Write to disk the original packets we captured - wrpcap(packets_path, [p.packet for p in self.seen_packets]) - - # If the engine exits before it initializes for any reason, these threads may not be set - # Only join them if they are defined - if self.out_nfqueue_thread: - self.out_nfqueue_thread.join() - if self.in_nfqueue_thread: - self.in_nfqueue_thread.join() - - # Shutdown the logger - actions.utils.close_logger(self.logger) - - def out_callback(self, nfpacket): - """ - Callback bound to the outgoing nfqueue rule to run the outbound strategy. - """ - if not self.running_nfqueue: - return - - packet = actions.packet.Packet(IP(nfpacket.get_payload())) + #print(divert_packet) + packet = actions.packet.Packet(IP(divert_packet.raw.tobytes())) + #print(packet.show2()) self.logger.debug("Received outbound packet %s", str(packet)) - # Record this packet for a .pacp later + # Record this packet for a .pcap later self.seen_packets.append(packet) - # Drop the packet in NFQueue so the strategy can handle it - nfpacket.drop() - - self.handle_packet(packet) - - def handle_packet(self, packet): - """ - Handles processing an outbound packet through the engine. - """ packets_to_send = self.strategy.act_on_packet(packet, self.logger, direction="out") # Send all of the packets we've collected to send for out_packet in packets_to_send: - # If the strategy requested us to sleep before sending this packet, do so here - if out_packet.sleep: - # We can't block the main sending thread, so instead spin off a new thread to handle sleeping - threading.Thread(target=self.delayed_send, args=(out_packet, out_packet.sleep)).start() - else: - self.mysend(out_packet) + self.mysend(out_packet, Direction.OUTBOUND) - def in_callback(self, nfpacket): + def handle_inbound_packet(self, divert_packet): """ - Callback bound to the incoming nfqueue rule. Since we can't - manually send packets to ourself, process the given packet here. + Handles inbound packets. Process the packet and forward it to the strategy if needed. """ - if not self.running_nfqueue: - return - packet = actions.packet.Packet(IP(nfpacket.get_payload())) + + packet = actions.packet.Packet(IP(divert_packet.raw.tobytes())) self.seen_packets.append(packet) @@ -284,20 +192,16 @@ class Engine(): self.censorship_detected = True # Branching is disabled for the in direction, so we can only ever get - # back 1 or 0 packets. If zero, drop the packet. + # back 1 or 0 packets. If zero, return and do not send packet. if not packets: - nfpacket.drop() return - # Otherwise, overwrite this packet with the packet the action trees gave back - nfpacket.set_payload(bytes(packets[0])) - # If the strategy requested us to sleep before accepting on this packet, do so here if packets[0].sleep: time.sleep(packets[0].sleep) # Accept the modified packet - nfpacket.accept() + self.mysend(packets[0], Direction.INBOUND) def get_args(): @@ -327,11 +231,11 @@ def main(args): environment_id=args.get("environment_id"), output_directory = args.get("output_directory"), log_level=args["log"]) - eng.initialize_nfqueue() + eng.initialize_divert() while True: time.sleep(0.5) finally: - eng.shutdown_nfqueue() + eng.shutdown_divert() if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 42fc3fe..3f2872a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ scapy==2.4.3 requests netifaces -netfilterqueue cryptography==2.5 requests anytree +pydivert \ No newline at end of file From 90edaff6baf67314bcaa74b6a8e9f5fdbf7e9c08 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Wed, 4 Dec 2019 00:18:38 -0800 Subject: [PATCH 02/13] First round of cleaning up, made engine more useable --- actions/utils.py | 24 +-- engine.py | 333 ++++++++++++++++++++++++++++++++++----- requirements.txt | 4 +- requirements_linux.txt | 7 + requirements_windows.txt | 7 + 5 files changed, 323 insertions(+), 52 deletions(-) create mode 100644 requirements_linux.txt create mode 100644 requirements_windows.txt diff --git a/actions/utils.py b/actions/utils.py index 97585e5..5520846 100644 --- a/actions/utils.py +++ b/actions/utils.py @@ -221,15 +221,15 @@ def get_interface(): """ Chooses an interface on the machine to use for socket testing. """ - """ - TODO: FIX - ifaces = netifaces.interfaces() - for iface in ifaces: - if "lo" in iface: - continue - info = netifaces.ifaddresses(iface) - # Filter for IPv4 addresses - if netifaces.AF_INET in info: - return iface - """ - return "Wi-Fi" + if os.name == 'nt': + # Windows code + return # TODO: Fix this + else: + ifaces = netifaces.interfaces() + for iface in ifaces: + if "lo" in iface: + continue + info = netifaces.ifaddresses(iface) + # Filter for IPv4 addresses + if netifaces.AF_INET in info: + return iface diff --git a/engine.py b/engine.py index a8f13e1..5c29029 100644 --- a/engine.py +++ b/engine.py @@ -18,9 +18,6 @@ from scapy.utils import wrpcap from scapy.config import conf from scapy.all import send, Raw -import pydivert #TODO -from pydivert.consts import Direction - socket.setdefaulttimeout(1) import actions.packet @@ -29,8 +26,16 @@ import actions.utils BASEPATH = os.path.dirname(os.path.abspath(__file__)) +if os.name == 'nt': + WINDOWS = True +else: + WINDOWS = False -class Engine(): +if WINDOWS: + import pydivert + from pydivert.consts import Direction + +class Engine: def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): self.server_port = server_port self.seen_packets = [] @@ -53,18 +58,16 @@ class Engine(): # Used for conditional context manager usage self.strategy = actions.utils.parse(string_strategy, self.logger) - + self.censorship_detected = False + +class WindowsEngine(Engine): + def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) # Instantialize a PyDivert channel, which we will use to redirect packets self.divert = None self.divert_thread = None self.divert_thread_started = False - - self.censorship_detected = False - # Specifically define an L3Socket to send our packets. This is an optimization - # for scapy to send packets more quickly than using just send(), as under the hood - # send() creates and then destroys a socket each time, imparting a large amount - # of overhead. - self.socket = conf.L3socket(iface=actions.utils.get_interface()) # TODO: FIX + self.interface = None # Using lazy evaluating as divert should know this def initialize_divert(self): """ @@ -99,8 +102,6 @@ class Engine(): if self.divert: self.divert.close() self.divert = None - - return def run_divert(self): """ @@ -110,6 +111,8 @@ class Engine(): self.divert_thread_started = True for packet in self.divert: + if not self.interface: + self.interface = packet.interface if packet.is_outbound: # Send to outbound action tree, if any self.handle_outbound_packet(packet) @@ -122,14 +125,17 @@ class Engine(): def __enter__(self): """ - TODO + Allows the engine to be used as a context manager; simply launches the + engine. """ + self.initialize_divert() return self def __exit__(self, exc_type, exc_value, tb): """ - TODO + Allows the engine to be used as a context manager; simply stops the engine """ + self.shutdown_divert() return def mysend(self, packet, dir): @@ -138,19 +144,10 @@ class Engine(): """ try: self.logger.debug("Sending packet %s", str(packet)) - - #Convert packet to pydivert - - #print(bytes(Raw(packet.packet))) - #print(packet.packet) - pack = bytes(packet.packet) - pack2 = bytearray(pack) - #print(pack2[0]) - #send(IP(packet.packet), iface="Wi-Fi") - #pack = bytearray(bytes(Raw(packet.packet)), "UTF-8") - #print(pack) - self.divert.send(pydivert.Packet(memoryview(pack2), (12, 0), dir), recalculate_checksum=False) # TODO: FIX - + # Convert the packet to a bytearray so memoryview can edit the underlying memory + pack = bytearray(bytes(packet.packet)) + # Don't recalculate checksum since sometimes we want to change it + self.divert.send(pydivert.Packet(memoryview(pack), self.interface, dir), recalculate_checksum=False) except Exception: self.logger.exception("Error in engine mysend.") @@ -158,9 +155,7 @@ class Engine(): """ Handles outbound packets by sending them the the strategy """ - #print(divert_packet) packet = actions.packet.Packet(IP(divert_packet.raw.tobytes())) - #print(packet.show2()) self.logger.debug("Received outbound packet %s", str(packet)) # Record this packet for a .pcap later @@ -203,6 +198,257 @@ class Engine(): # Accept the modified packet self.mysend(packets[0], Direction.INBOUND) +class LinuxEngine(Engine): + def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + + super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) + + # Setup variables used by the NFQueue system + self.out_nfqueue_started = False + self.in_nfqueue_started = False + self.running_nfqueue = False + self.out_nfqueue = None + self.in_nfqueue = None + self.out_nfqueue_socket = None + self.in_nfqueue_socket = None + self.out_nfqueue_thread = None + self.in_nfqueue_thread = None + + # Specifically define an L3Socket to send our packets. This is an optimization + # for scapy to send packets more quickly than using just send(), as under the hood + # send() creates and then destroys a socket each time, imparting a large amount + # of overhead. + self.socket = conf.L3socket(iface=actions.utils.get_interface()) + + def __enter__(self): + """ + Allows the engine to be used as a context manager; simply launches the + engine. + """ + self.initialize_nfqueue() + return self + + def __exit__(self, exc_type, exc_value, tb): + """ + Allows the engine to be used as a context manager; simply stops the engine + """ + self.shutdown_nfqueue() + + def mysend(self, packet): + """ + Helper scapy sending method. Expects a Geneva Packet input. + """ + try: + self.logger.debug("Sending packet %s", str(packet)) + self.socket.send(packet.packet) + except Exception: + self.logger.exception("Error in engine mysend.") + + def delayed_send(self, packet, delay): + """ + Method to be started by a thread to delay the sending of a packet without blocking the main thread. + """ + self.logger.debug("Sleeping for %f seconds." % delay) + time.sleep(delay) + self.mysend(packet) + + def run_nfqueue(self, nfqueue, nfqueue_socket, direction): + """ + Handles running the outbound nfqueue socket with the socket timeout. + """ + try: + while self.running_nfqueue: + try: + if direction == "out": + self.out_nfqueue_started = True + else: + self.in_nfqueue_started = True + + nfqueue.run_socket(nfqueue_socket) + except socket.timeout: + pass + except Exception: + self.logger.exception("Exception out of run_nfqueue() (direction=%s)", direction) + + def configure_iptables(self, remove=False): + """ + Handles setting up ipables for this run + """ + self.logger.debug("Configuring iptables rules") + + port1, port2 = "dport", "sport" + + out_chain = "OUTPUT" + in_chain = "INPUT" + + # Switch whether the command should add or delete the rules + add_or_remove = "A" + if remove: + add_or_remove = "D" + cmds = [] + for proto in ["tcp", "udp"]: + cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 1" % + (add_or_remove, out_chain, proto, port1, self.server_port), + "iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 2" % + (add_or_remove, in_chain, proto, port2, self.server_port)] + + for cmd in cmds: + self.logger.debug(cmd) + # If we're logging at DEBUG mode, keep stderr/stdout piped to us + # Otherwise, pipe them both to DEVNULL + if actions.utils.get_console_log_level() == logging.DEBUG: + subprocess.check_call(cmd.split(), timeout=60) + else: + subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) + return cmds + + def initialize_nfqueue(self): + """ + Initializes the nfqueue for input and output forests. + """ + self.logger.debug("Engine created with strategy %s (ID %s) to port %s", + str(self.strategy).strip(), self.environment_id, self.server_port) + self.configure_iptables() + + self.out_nfqueue_started = False + self.in_nfqueue_started = False + self.running_nfqueue = True + # Create our NFQueues + self.out_nfqueue = netfilterqueue.NetfilterQueue() + self.in_nfqueue = netfilterqueue.NetfilterQueue() + # Bind them + self.out_nfqueue.bind(1, self.out_callback) + self.in_nfqueue.bind(2, self.in_callback) + # Create our nfqueue sockets to allow for non-blocking usage + self.out_nfqueue_socket = socket.fromfd(self.out_nfqueue.get_fd(), + socket.AF_UNIX, + socket.SOCK_STREAM) + self.in_nfqueue_socket = socket.fromfd(self.in_nfqueue.get_fd(), + socket.AF_UNIX, + socket.SOCK_STREAM) + # Create our handling threads for packets + self.out_nfqueue_thread = threading.Thread(target=self.run_nfqueue, + args=(self.out_nfqueue, self.out_nfqueue_socket, "out")) + + self.in_nfqueue_thread = threading.Thread(target=self.run_nfqueue, + args=(self.in_nfqueue, self.in_nfqueue_socket, "in")) + # Start each thread + self.in_nfqueue_thread.start() + self.out_nfqueue_thread.start() + + maxwait = 100 # 100 time steps of 0.01 seconds for a max wait of 10 seconds + i = 0 + # Give NFQueue time to startup, since it's running in background threads + # Block the main thread until this is done + while (not self.in_nfqueue_started or not self.out_nfqueue_started) and i < maxwait: + time.sleep(0.1) + i += 1 + self.logger.debug("NFQueue Initialized after %d", int(i)) + + def shutdown_nfqueue(self): + """ + Shutdown nfqueue. + """ + self.logger.debug("Shutting down NFQueue") + self.out_nfqueue_started = False + self.in_nfqueue_started = False + self.running_nfqueue = False + # Give the handlers two seconds to leave the callbacks before we forcibly unbind + # the queues. + time.sleep(2) + if self.in_nfqueue: + self.in_nfqueue.unbind() + if self.out_nfqueue: + self.out_nfqueue.unbind() + self.configure_iptables(remove=True) + + packets_path = os.path.join(BASEPATH, + self.output_directory, + "packets", + "original_%s.pcap" % self.environment_id) + + # Write to disk the original packets we captured + wrpcap(packets_path, [p.packet for p in self.seen_packets]) + + # If the engine exits before it initializes for any reason, these threads may not be set + # Only join them if they are defined + if self.out_nfqueue_thread: + self.out_nfqueue_thread.join() + if self.in_nfqueue_thread: + self.in_nfqueue_thread.join() + + # Shutdown the logger + actions.utils.close_logger(self.logger) + + def out_callback(self, nfpacket): + """ + Callback bound to the outgoing nfqueue rule to run the outbound strategy. + """ + if not self.running_nfqueue: + return + + packet = actions.packet.Packet(IP(nfpacket.get_payload())) + self.logger.debug("Received outbound packet %s", str(packet)) + + # Record this packet for a .pacp later + self.seen_packets.append(packet) + + # Drop the packet in NFQueue so the strategy can handle it + nfpacket.drop() + + self.handle_packet(packet) + + def handle_packet(self, packet): + """ + Handles processing an outbound packet through the engine. + """ + packets_to_send = self.strategy.act_on_packet(packet, self.logger, direction="out") + + # Send all of the packets we've collected to send + for out_packet in packets_to_send: + # If the strategy requested us to sleep before sending this packet, do so here + if out_packet.sleep: + # We can't block the main sending thread, so instead spin off a new thread to handle sleeping + threading.Thread(target=self.delayed_send, args=(out_packet, out_packet.sleep)).start() + else: + self.mysend(out_packet) + + def in_callback(self, nfpacket): + """ + Callback bound to the incoming nfqueue rule. Since we can't + manually send packets to ourself, process the given packet here. + """ + if not self.running_nfqueue: + return + packet = actions.packet.Packet(IP(nfpacket.get_payload())) + + self.seen_packets.append(packet) + + self.logger.debug("Received packet: %s", str(packet)) + + # Run the given strategy + packets = self.strategy.act_on_packet(packet, self.logger, direction="in") + + # GFW will send RA packets to disrupt a TCP stream + if packet.haslayer("TCP") and packet.get("TCP", "flags") == "RA": + self.logger.debug("Detected GFW censorship - strategy failed.") + self.censorship_detected = True + + # Branching is disabled for the in direction, so we can only ever get + # back 1 or 0 packets. If zero, drop the packet. + if not packets: + nfpacket.drop() + return + + # Otherwise, overwrite this packet with the packet the action trees gave back + nfpacket.set_payload(bytes(packets[0])) + + # If the strategy requested us to sleep before accepting on this packet, do so here + if packets[0].sleep: + time.sleep(packets[0].sleep) + + # Accept the modified packet + nfpacket.accept() def get_args(): """ @@ -226,16 +472,27 @@ def main(args): Kicks off the engine with the given arguments. """ try: - eng = Engine(args["server_port"], - args["strategy"], - environment_id=args.get("environment_id"), - output_directory = args.get("output_directory"), - log_level=args["log"]) - eng.initialize_divert() + if WINDOWS: + eng = WindowsEngine(args["server_port"], + args["strategy"], + environment_id=args.get("environment_id"), + output_directory = args.get("output_directory"), + log_level=args["log"]) + eng.initialize_divert() + else: + eng = LinuxEngine(args["server_port"], + args["strategy"], + environment_id=args.get("environment_id"), + output_directory = args.get("output_directory"), + log_level=args["log"]) + eng.initialize_nfqueue() while True: time.sleep(0.5) finally: - eng.shutdown_divert() + if WINDOWS: + eng.shutdown_divert() + else: + eng.shutdown_nfqueue() if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 3f2872a..0840c7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ scapy==2.4.3 requests netifaces +netfilterqueue cryptography==2.5 requests -anytree -pydivert \ No newline at end of file +anytree \ No newline at end of file diff --git a/requirements_linux.txt b/requirements_linux.txt new file mode 100644 index 0000000..0840c7a --- /dev/null +++ b/requirements_linux.txt @@ -0,0 +1,7 @@ +scapy==2.4.3 +requests +netifaces +netfilterqueue +cryptography==2.5 +requests +anytree \ No newline at end of file diff --git a/requirements_windows.txt b/requirements_windows.txt new file mode 100644 index 0000000..3f2872a --- /dev/null +++ b/requirements_windows.txt @@ -0,0 +1,7 @@ +scapy==2.4.3 +requests +netifaces +cryptography==2.5 +requests +anytree +pydivert \ No newline at end of file From b85fd72be04896ebd4982f21ba5ba02de605580d Mon Sep 17 00:00:00 2001 From: George Hughey Date: Wed, 4 Dec 2019 00:20:04 -0800 Subject: [PATCH 03/13] Readded import --- engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine.py b/engine.py index 5c29029..d3314c4 100644 --- a/engine.py +++ b/engine.py @@ -34,6 +34,8 @@ else: if WINDOWS: import pydivert from pydivert.consts import Direction +else: + import netfilterqueue class Engine: def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): From 97f398d3b60659c0cc8a8055576425ec85edcfb8 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Mon, 9 Dec 2019 19:21:51 -0800 Subject: [PATCH 04/13] Removed requirements, update readme --- README.md | 14 +++++++++----- requirements.txt | 7 ------- 2 files changed, 9 insertions(+), 12 deletions(-) delete mode 100644 requirements.txt diff --git a/README.md b/README.md index e2e2804..e6a7302 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,21 @@ This code release specifically contains the strategy engine used by Geneva, its ## Setup -Geneva has been developed and tested for Centos or Debian-based systems. Due to limitations of -netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6* (with more versions coming soon). +Geneva has been developed and tested for Centos or Debian-based systems. Windows support is currently in beta and requires more testing, but is available in this repository. Due to limitations of netfilter and raw sockets, Geneva does not work on OS X at this time and requires *python3.6* on Linux (with more versions coming soon). -Install netfilterqueue dependencies: +Install netfilterqueue dependencies (Linux): ``` # sudo apt-get install build-essential python-dev libnetfilter-queue-dev libffi-dev libssl-dev iptables python3-pip ``` -Install Python dependencies: +Install Python dependencies (Linux): ``` -# python3 -m pip install -r requirements.txt +# python3 -m pip install -r requirements_linux.txt +``` + +Install Python dependencies (Windows): +``` +# python3 -m pip install -r requirements_windows.txt ``` ## Running it diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0840c7a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -scapy==2.4.3 -requests -netifaces -netfilterqueue -cryptography==2.5 -requests -anytree \ No newline at end of file From 556f94380d62f6711c5d8f9e8ceedafce32765c5 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Mon, 9 Dec 2019 23:36:56 -0800 Subject: [PATCH 05/13] Small cleanup --- engine.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/engine.py b/engine.py index d3314c4..582bc61 100644 --- a/engine.py +++ b/engine.py @@ -123,8 +123,6 @@ class WindowsEngine(Engine): # Send to inbound action tree, if any self.handle_inbound_packet(packet) - return - def __enter__(self): """ Allows the engine to be used as a context manager; simply launches the @@ -138,7 +136,6 @@ class WindowsEngine(Engine): Allows the engine to be used as a context manager; simply stops the engine """ self.shutdown_divert() - return def mysend(self, packet, dir): """ @@ -148,7 +145,7 @@ class WindowsEngine(Engine): self.logger.debug("Sending packet %s", str(packet)) # Convert the packet to a bytearray so memoryview can edit the underlying memory pack = bytearray(bytes(packet.packet)) - # Don't recalculate checksum since sometimes we want to change it + # Don't recalculate checksum since sometimes we will have already changed it self.divert.send(pydivert.Packet(memoryview(pack), self.interface, dir), recalculate_checksum=False) except Exception: self.logger.exception("Error in engine mysend.") From 5c4a4ea08ddb865b72e55ce345cca637681b34b4 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Thu, 12 Dec 2019 16:44:24 -0800 Subject: [PATCH 06/13] Add strategy num --- engine.py | 14 ++++++++++++-- library.py | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 library.py diff --git a/engine.py b/engine.py index 582bc61..71ca84f 100644 --- a/engine.py +++ b/engine.py @@ -18,6 +18,8 @@ from scapy.utils import wrpcap from scapy.config import conf from scapy.all import send, Raw +from library import LIBRARY + socket.setdefaulttimeout(1) import actions.packet @@ -457,6 +459,7 @@ def get_args(): parser.add_argument('--server-port', type=int, action='store', required=True) parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test. If not provided, one will be generated.") parser.add_argument('--strategy', action='store', help="Strategy to deploy") + parser.add_argument('--strategy-index', action='store', help="Strategy to deploy, specified by index in the library") parser.add_argument('--output-directory', default="trials", action='store', help="Where to output logs, captures, and results. Defaults to trials/.") parser.add_argument('--log', action='store', default="debug", choices=("debug", "info", "warning", "critical", "error"), @@ -471,16 +474,23 @@ def main(args): Kicks off the engine with the given arguments. """ try: + if args["strategy"]: + strategy = args["strategy"] + elif args["strategy-index"]: + strategy = LIBRARY[int(args["strategy-index"])][0] + else: + # Default to first strategy + strategy = LIBRARY[0][0] if WINDOWS: eng = WindowsEngine(args["server_port"], - args["strategy"], + strategy, environment_id=args.get("environment_id"), output_directory = args.get("output_directory"), log_level=args["log"]) eng.initialize_divert() else: eng = LinuxEngine(args["server_port"], - args["strategy"], + strategy, environment_id=args.get("environment_id"), output_directory = args.get("output_directory"), log_level=args["log"]) diff --git a/library.py b/library.py new file mode 100644 index 0000000..f4dab66 --- /dev/null +++ b/library.py @@ -0,0 +1,26 @@ +LIBRARY = [ + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|", 98%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-|", 98%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-|", 94%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-|", 98%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-|", 80%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8},),)-|", 98%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt},),)-|", 87%, 100%, 0%), + ("[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|", 3%, 100%, 0%), + ("[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|", 3%, 0%, 100%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-|", 95%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-|", 87%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 86%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:corrupt},))-|", 80%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-|", 94%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 94%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))-|", 89%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-|", 96%, 0%, 0%), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-|", 94%, 0%, 0%), + ("[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:corrupt}-|", 94%, 100%, 100%), + ("[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-|", 98%, 100%, 100%), + ("[TCP:flags:PA]-fragment{tcp:-1:True}-|", 3%, 100%, 100%), + ("[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:78},),)-|", 53%, 0%, 100%), + ("[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|", 3%, 100%, 0%), + ("[TCP:flags:PA]-tamper{TCP:options-uto:corrupt}-|", 3%, 0%, 100%) +] \ No newline at end of file From 4f1f70e0b3718a41dedf279d66d106155d463be0 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Thu, 12 Dec 2019 18:30:58 -0800 Subject: [PATCH 07/13] Made Engine more user friendly with ABC --- engine.py | 125 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/engine.py b/engine.py index 71ca84f..2fa34af 100644 --- a/engine.py +++ b/engine.py @@ -39,8 +39,31 @@ if WINDOWS: else: import netfilterqueue -class Engine: +from abc import ABC, abstractmethod + +def Engine(server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + # Factory function to dynamically choose which engine to use. + # Users should initialize an Engine using this. + if WINDOWS: + eng = WindowsEngine(server_port, + string_strategy, + environment_id=environment_id, + output_directory=output_directory, + log_level=log_level) + else: + eng = LinuxEngine(server_port, + string_strategy, + environment_id=environment_id, + output_directory=output_directory, + log_level=log_level) + + return eng + +class GenericEngine(ABC): + # Abstract Base Class defining an engine. + # Users should follow the contract laid out here to create custom engines. def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + # Do common setup self.server_port = server_port self.seen_packets = [] # Set up the directory and ID for logging @@ -53,18 +76,42 @@ class Engine: self.environment_id = environment_id # Set up a logger self.logger = actions.utils.get_logger(BASEPATH, - output_directory, - __name__, - "engine", - environment_id, - log_level=log_level) + output_directory, + __name__, + "engine", + environment_id, + log_level=log_level) self.output_directory = output_directory # Used for conditional context manager usage self.strategy = actions.utils.parse(string_strategy, self.logger) self.censorship_detected = False -class WindowsEngine(Engine): + @abstractmethod + def initialize(self): + # Initialize the Engine. Users should call this directly. + pass + + @abstractmethod + def shutdown(self): + # Clean up the Engine. Users should call this directly. + pass + + def __enter__(self): + """ + Allows the engine to be used as a context manager; simply launches the + engine. + """ + self.initialize() + return self + + def __exit__(self, exc_type, exc_value, tb): + """ + Allows the engine to be used as a context manager; simply stops the engine + """ + self.shutdown() + +class WindowsEngine(GenericEngine): def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) # Instantialize a PyDivert channel, which we will use to redirect packets @@ -73,7 +120,7 @@ class WindowsEngine(Engine): self.divert_thread_started = False self.interface = None # Using lazy evaluating as divert should know this - def initialize_divert(self): + def initialize(self): """ Initializes Divert such that all packets for the connection will come through us """ @@ -99,7 +146,7 @@ class WindowsEngine(Engine): return - def shutdown_divert(self): + def shutdown(self): """ Closes the divert connection """ @@ -124,21 +171,7 @@ class WindowsEngine(Engine): elif packet.is_inbound: # Send to inbound action tree, if any self.handle_inbound_packet(packet) - - def __enter__(self): - """ - Allows the engine to be used as a context manager; simply launches the - engine. - """ - self.initialize_divert() - return self - - def __exit__(self, exc_type, exc_value, tb): - """ - Allows the engine to be used as a context manager; simply stops the engine - """ - self.shutdown_divert() - + def mysend(self, packet, dir): """ Helper scapy sending method. Expects a Geneva Packet input. @@ -199,11 +232,9 @@ class WindowsEngine(Engine): # Accept the modified packet self.mysend(packets[0], Direction.INBOUND) -class LinuxEngine(Engine): +class LinuxEngine(GenericEngine): def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): - super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) - # Setup variables used by the NFQueue system self.out_nfqueue_started = False self.in_nfqueue_started = False @@ -221,19 +252,10 @@ class LinuxEngine(Engine): # of overhead. self.socket = conf.L3socket(iface=actions.utils.get_interface()) - def __enter__(self): - """ - Allows the engine to be used as a context manager; simply launches the - engine. - """ - self.initialize_nfqueue() - return self - - def __exit__(self, exc_type, exc_value, tb): """ Allows the engine to be used as a context manager; simply stops the engine """ - self.shutdown_nfqueue() + self.shutdown() def mysend(self, packet): """ @@ -303,7 +325,7 @@ class LinuxEngine(Engine): subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) return cmds - def initialize_nfqueue(self): + def initialize(self): """ Initializes the nfqueue for input and output forests. """ @@ -346,7 +368,7 @@ class LinuxEngine(Engine): i += 1 self.logger.debug("NFQueue Initialized after %d", int(i)) - def shutdown_nfqueue(self): + def shutdown(self): """ Shutdown nfqueue. """ @@ -473,36 +495,27 @@ def main(args): """ Kicks off the engine with the given arguments. """ + try: if args["strategy"]: strategy = args["strategy"] - elif args["strategy-index"]: - strategy = LIBRARY[int(args["strategy-index"])][0] + elif args["strategy_index"]: + strategy = LIBRARY[int(args["strategy_index"])][0] else: # Default to first strategy strategy = LIBRARY[0][0] - if WINDOWS: - eng = WindowsEngine(args["server_port"], + eng = Engine(args["server_port"], strategy, environment_id=args.get("environment_id"), output_directory = args.get("output_directory"), log_level=args["log"]) - eng.initialize_divert() - else: - eng = LinuxEngine(args["server_port"], - strategy, - environment_id=args.get("environment_id"), - output_directory = args.get("output_directory"), - log_level=args["log"]) - eng.initialize_nfqueue() + eng.initialize() while True: time.sleep(0.5) + except Exception as e: + print(e) finally: - if WINDOWS: - eng.shutdown_divert() - else: - eng.shutdown_nfqueue() - + eng.shutdown() if __name__ == "__main__": main(vars(get_args())) From abd3cdc2f526de3b2edbf49e3f5ccb9e3a372634 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Thu, 12 Dec 2019 18:33:20 -0800 Subject: [PATCH 08/13] Fixed library --- library.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/library.py b/library.py index f4dab66..b95d6c3 100644 --- a/library.py +++ b/library.py @@ -1,26 +1,26 @@ LIBRARY = [ - ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|", 98%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-|", 98%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-|", 94%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-|", 98%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-|", 80%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8},),)-|", 98%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt},),)-|", 87%, 100%, 0%), - ("[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|", 3%, 100%, 0%), - ("[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|", 3%, 0%, 100%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-|", 95%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-|", 87%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 86%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:corrupt},))-|", 80%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-|", 94%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 94%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))-|", 89%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-|", 96%, 0%, 0%), - ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-|", 94%, 0%, 0%), - ("[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:corrupt}-|", 94%, 100%, 100%), - ("[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-|", 98%, 100%, 100%), - ("[TCP:flags:PA]-fragment{tcp:-1:True}-|", 3%, 100%, 100%), - ("[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:78},),)-|", 53%, 0%, 100%), - ("[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|", 3%, 100%, 0%), - ("[TCP:flags:PA]-tamper{TCP:options-uto:corrupt}-|", 3%, 0%, 100%) + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-|", 94, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-|", 80, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt},),)-|", 87, 100, 0), + ("[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|", 3, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|", 3, 0, 100), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-|", 95, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-|", 87, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 86, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:corrupt},))-|", 80, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-|", 94, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 94, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))-|", 89, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-|", 96, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-|", 94, 0, 0), + ("[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:corrupt}-|", 94, 100, 100), + ("[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-|", 98, 100, 100), + ("[TCP:flags:PA]-fragment{tcp:-1:True}-|", 3, 100, 100), + ("[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:78},),)-|", 53, 0, 100), + ("[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|", 3, 100, 0), + ("[TCP:flags:PA]-tamper{TCP:options-uto:corrupt}-|", 3, 0, 100) ] \ No newline at end of file From 7b983670651d961fbdfcd7451a12ea132fe7c35d Mon Sep 17 00:00:00 2001 From: George Hughey Date: Thu, 12 Dec 2019 21:57:09 -0800 Subject: [PATCH 09/13] Fixed whitespace --- engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/engine.py b/engine.py index 2fa34af..d3b5879 100644 --- a/engine.py +++ b/engine.py @@ -490,7 +490,6 @@ def get_args(): args = parser.parse_args() return args - def main(args): """ Kicks off the engine with the given arguments. From c8250db14338c3a54e054d7a1c5635617e7c76f4 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Sat, 14 Dec 2019 00:37:24 -0800 Subject: [PATCH 10/13] Initial commit of overlapping segments --- actions/fragment.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/actions/fragment.py b/actions/fragment.py index e16a94e..e26cfdf 100644 --- a/actions/fragment.py +++ b/actions/fragment.py @@ -6,7 +6,7 @@ from scapy.all import IP, TCP, fragment class FragmentAction(Action): - def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True): + def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True, overlap=0): ''' correct_order specifies if the fragmented packets should come in the correct order fragsize specifies how @@ -17,6 +17,7 @@ class FragmentAction(Action): self.terminal = False self.fragsize = fragsize self.segment = segment + self.overlap = overlap if correct_order == None: self.correct_order = self.get_rand_order() @@ -87,6 +88,9 @@ class FragmentAction(Action): Segments a packet into two, given the size of the first packet (0:fragsize) Always returns two packets, since fragment is a branching action, so if we are unable to segment, it will duplicate the packet. + + If overlap is specified, it will select n bytes from the second packet + and append them to the first, and increment the sequence number accordingly """ if not packet.haslayer("TCP") or not hasattr(packet["TCP"], "load") or not packet["TCP"].load: return packet, packet.copy() # duplicate if no TCP or no payload to segment @@ -101,7 +105,11 @@ class FragmentAction(Action): fragsize = int(len(payload)/2) # Craft new packets - pkt1 = IP(packet["IP"])/payload[:fragsize] + + # Make sure we don't go out of bounds by choosing the min + overlapBytes = min(payload[fragsize:], overlap) + # Attach these bytes to the first packet + pkt1 = IP(packet["IP"])/payload[:fragsize + overlapBytes] pkt2 = IP(packet["IP"])/payload[fragsize:] # We cannot rely on scapy's native parsing here - if a previous action has changed the From 61acc0970da8028f22204d1379190efa125c492d Mon Sep 17 00:00:00 2001 From: George Hughey Date: Sun, 15 Dec 2019 17:21:17 -0800 Subject: [PATCH 11/13] Add overlapping segmentation --- actions/fragment.py | 39 ++++++++++++++++------ tests/test_fragment.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/actions/fragment.py b/actions/fragment.py index a0701c8..d47f37b 100644 --- a/actions/fragment.py +++ b/actions/fragment.py @@ -107,7 +107,7 @@ class FragmentAction(Action): # Craft new packets # Make sure we don't go out of bounds by choosing the min - overlapBytes = min(payload[fragsize:], overlap) + overlapBytes = min(len(payload[fragsize:]), self.overlap) # Attach these bytes to the first packet pkt1 = IP(packet["IP"])/payload[:fragsize + overlapBytes] pkt2 = IP(packet["IP"])/payload[fragsize:] @@ -155,10 +155,15 @@ class FragmentAction(Action): Returns a string representation with the fragsize """ s = Action.__str__(self) - if self.segment: - s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + "}" + if self.overlap == 0: + ending = "}" else: - s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + "}" + ending = ":" + str(self.overlap) + "}" + + if self.segment: + s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + ending + else: + s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + ending return s def parse(self, string, logger): @@ -177,22 +182,36 @@ class FragmentAction(Action): num_parameters = string.count(":") # If num_parameters is greater than 2, it's not a valid fragment action - if num_parameters != 2: - msg = "Cannot parse fragment action %s" % string - logger.error(msg) - raise Exception(msg) - else: + if num_parameters == 2: params = string.split(":") seg, fragsize, correct_order = params + overlap = 0 if "tcp" in seg: self.segment = True else: self.segment = False + elif num_parameters == 3: + params = string.split(":") + seg, fragsize, correct_order, overlap = params + if overlap.endswith("}"): + overlap = overlap[:-1] # Chop off trailing } + if "tcp" in seg: + self.segment = True + else: + self.segment = False + + else: + msg = "Cannot parse fragment action %s" % string + logger.error(msg) + raise Exception(msg) + try: # Try to convert to int self.fragsize = int(fragsize) - except ValueError: + self.overlap = int(overlap) + except ValueError as e: + print(e) msg = "Cannot parse fragment action %s" % string logger.error(msg) raise Exception(msg) diff --git a/tests/test_fragment.py b/tests/test_fragment.py index 3e7bcee..5beaec3 100644 --- a/tests/test_fragment.py +++ b/tests/test_fragment.py @@ -217,4 +217,78 @@ def test_ip_only_fragment(): assert packet1["Raw"].load == b'datadata', "Left packet incorrectly fragmented" assert packet2["Raw"].load == b"11datadata", "Right packet incorrectly fragmented" +def test_overlapping_segment(): + """ + Basic test for overlapping segments. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:4}", logger) + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11dat', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_no_overlap(): + """ + Basic test for overlapping segments with no overlap. (shouldn't ever actually happen) + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:0}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata1', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_entire_packet(): + """ + Basic test for overlapping segments overlapping entire packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:9}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_out_of_bounds(): + """ + Basic test for overlapping segments overlapping beyond the edge of the packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:20}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" \ No newline at end of file From 25014462f01e541f898646b58b9039e274fbb21d Mon Sep 17 00:00:00 2001 From: George Hughey Date: Sun, 15 Dec 2019 17:37:43 -0800 Subject: [PATCH 12/13] Add parsing test for overlapping segment --- tests/test_fragment.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_fragment.py b/tests/test_fragment.py index 5beaec3..03a1c64 100644 --- a/tests/test_fragment.py +++ b/tests/test_fragment.py @@ -291,4 +291,12 @@ def test_overlapping_segment_out_of_bounds(): assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" - assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" \ No newline at end of file + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segmentation_parse(): + """ + Basic test for parsing overlapping segments. + """ + + fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=2, segment=True, overlap=3) + assert str(fragment) == "fragment{tcp:2:False:3}", "Fragment returned incorrect string representation: %s" % str(fragment) From 337f916920637eb392220ab58a543894c08a9de7 Mon Sep 17 00:00:00 2001 From: George Hughey Date: Sun, 15 Dec 2019 17:40:42 -0800 Subject: [PATCH 13/13] Let travis test for Linux --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9e296ac..92f4803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ install: # Copy in the sudoers file - sudo cp /tmp/sudoers.tmp /etc/sudoers # Now that sudo is good to go, finish installing dependencies - - sudo python3 -m pip install -r requirements.txt + - sudo python3 -m pip install -r requirements_linux.txt - sudo python3 -m pip install slackclient pytest-cov script: