diff --git a/.travis.yml b/.travis.yml index 9e296ac..92f4803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ install: # Copy in the sudoers file - sudo cp /tmp/sudoers.tmp /etc/sudoers # Now that sudo is good to go, finish installing dependencies - - sudo python3 -m pip install -r requirements.txt + - sudo python3 -m pip install -r requirements_linux.txt - sudo python3 -m pip install slackclient pytest-cov script: diff --git a/README.md b/README.md index 14075b2..f04ae1b 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,21 @@ This code release specifically contains the strategy engine used by Geneva, its ## Setup -Geneva has been developed and tested for Centos or Debian-based systems. Due to limitations of -netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6* (with more versions coming soon). +Geneva has been developed and tested for Centos or Debian-based systems. Windows support is currently in beta and requires more testing, but is available in this repository. Due to limitations of netfilter and raw sockets, Geneva does not work on OS X at this time and requires *python3.6* on Linux (with more versions coming soon). -Install netfilterqueue dependencies: +Install netfilterqueue dependencies (Linux): ``` # sudo apt-get install build-essential python-dev libnetfilter-queue-dev libffi-dev libssl-dev iptables python3-pip ``` -Install Python dependencies: +Install Python dependencies (Linux): ``` -# python3 -m pip install -r requirements.txt +# python3 -m pip install -r requirements_linux.txt +``` + +Install Python dependencies (Windows): +``` +# python3 -m pip install -r requirements_windows.txt ``` ## Running it diff --git a/actions/fragment.py b/actions/fragment.py index d7f80c3..d47f37b 100644 --- a/actions/fragment.py +++ b/actions/fragment.py @@ -6,7 +6,7 @@ from scapy.all import IP, TCP, fragment class FragmentAction(Action): - def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True): + def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True, overlap=0): ''' correct_order specifies if the fragmented packets should come in the correct order fragsize specifies how @@ -17,6 +17,7 @@ class FragmentAction(Action): self.terminal = False self.fragsize = fragsize self.segment = segment + self.overlap = overlap if correct_order == None: self.correct_order = self.get_rand_order() @@ -87,6 +88,9 @@ class FragmentAction(Action): Segments a packet into two, given the size of the first packet (0:fragsize) Always returns two packets, since fragment is a branching action, so if we are unable to segment, it will duplicate the packet. + + If overlap is specified, it will select n bytes from the second packet + and append them to the first, and increment the sequence number accordingly """ if not packet.haslayer("TCP") or not hasattr(packet["TCP"], "load") or not packet["TCP"].load: return packet, packet.copy() # duplicate if no TCP or no payload to segment @@ -101,7 +105,11 @@ class FragmentAction(Action): fragsize = int(len(payload)/2) # Craft new packets - pkt1 = IP(packet["IP"])/payload[:fragsize] + + # Make sure we don't go out of bounds by choosing the min + overlapBytes = min(len(payload[fragsize:]), self.overlap) + # Attach these bytes to the first packet + pkt1 = IP(packet["IP"])/payload[:fragsize + overlapBytes] pkt2 = IP(packet["IP"])/payload[fragsize:] # We cannot rely on scapy's native parsing here - if a previous action has changed the @@ -147,10 +155,15 @@ class FragmentAction(Action): Returns a string representation with the fragsize """ s = Action.__str__(self) - if self.segment: - s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + "}" + if self.overlap == 0: + ending = "}" else: - s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + "}" + ending = ":" + str(self.overlap) + "}" + + if self.segment: + s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + ending + else: + s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + ending return s def parse(self, string, logger): @@ -169,22 +182,36 @@ class FragmentAction(Action): num_parameters = string.count(":") # If num_parameters is greater than 2, it's not a valid fragment action - if num_parameters != 2: - msg = "Cannot parse fragment action %s" % string - logger.error(msg) - raise Exception(msg) - else: + if num_parameters == 2: params = string.split(":") seg, fragsize, correct_order = params + overlap = 0 if "tcp" in seg: self.segment = True else: self.segment = False + elif num_parameters == 3: + params = string.split(":") + seg, fragsize, correct_order, overlap = params + if overlap.endswith("}"): + overlap = overlap[:-1] # Chop off trailing } + if "tcp" in seg: + self.segment = True + else: + self.segment = False + + else: + msg = "Cannot parse fragment action %s" % string + logger.error(msg) + raise Exception(msg) + try: # Try to convert to int self.fragsize = int(fragsize) - except ValueError: + self.overlap = int(overlap) + except ValueError as e: + print(e) msg = "Cannot parse fragment action %s" % string logger.error(msg) raise Exception(msg) diff --git a/actions/utils.py b/actions/utils.py index 61db62b..a795442 100644 --- a/actions/utils.py +++ b/actions/utils.py @@ -181,11 +181,15 @@ def get_interface(): """ Chooses an interface on the machine to use for socket testing. """ - ifaces = netifaces.interfaces() - for iface in ifaces: - if "lo" in iface: - continue - info = netifaces.ifaddresses(iface) - # Filter for IPv4 addresses - if netifaces.AF_INET in info: - return iface + if os.name == 'nt': + # Windows code + return # TODO: Fix this + else: + ifaces = netifaces.interfaces() + for iface in ifaces: + if "lo" in iface: + continue + info = netifaces.ifaddresses(iface) + # Filter for IPv4 addresses + if netifaces.AF_INET in info: + return iface diff --git a/engine.py b/engine.py index f76cdc3..0f6cf08 100644 --- a/engine.py +++ b/engine.py @@ -4,7 +4,6 @@ Engine Given a strategy and a server port, the engine configures NFQueue so the strategy can run on the underlying connection. """ - import argparse import logging logging.getLogger("scapy.runtime").setLevel(logging.ERROR) @@ -14,11 +13,12 @@ import subprocess import threading import time -import netfilterqueue - from scapy.layers.inet import IP from scapy.utils import wrpcap from scapy.config import conf +from scapy.all import send, Raw + +from library import LIBRARY socket.setdefaulttimeout(1) @@ -28,9 +28,42 @@ import actions.utils BASEPATH = os.path.dirname(os.path.abspath(__file__)) +if os.name == 'nt': + WINDOWS = True +else: + WINDOWS = False -class Engine(): +if WINDOWS: + import pydivert + from pydivert.consts import Direction +else: + import netfilterqueue + +from abc import ABC, abstractmethod + +def Engine(server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + # Factory function to dynamically choose which engine to use. + # Users should initialize an Engine using this. + if WINDOWS: + eng = WindowsEngine(server_port, + string_strategy, + environment_id=environment_id, + output_directory=output_directory, + log_level=log_level) + else: + eng = LinuxEngine(server_port, + string_strategy, + environment_id=environment_id, + output_directory=output_directory, + log_level=log_level) + + return eng + +class GenericEngine(ABC): + # Abstract Base Class defining an engine. + # Users should follow the contract laid out here to create custom engines. def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + # Do common setup self.server_port = server_port self.seen_packets = [] # Set up the directory and ID for logging @@ -41,15 +74,165 @@ class Engine(): self.environment_id = environment_id # Set up a logger self.logger = actions.utils.get_logger(BASEPATH, - output_directory, - __name__, - "engine", - environment_id, - log_level=log_level) + output_directory, + __name__, + "engine", + environment_id, + log_level=log_level) self.output_directory = output_directory # Used for conditional context manager usage self.strategy = actions.utils.parse(string_strategy, self.logger) + self.censorship_detected = False + + @abstractmethod + def initialize(self): + # Initialize the Engine. Users should call this directly. + pass + + @abstractmethod + def shutdown(self): + # Clean up the Engine. Users should call this directly. + pass + + def __enter__(self): + """ + Allows the engine to be used as a context manager; simply launches the + engine. + """ + self.initialize() + return self + + def __exit__(self, exc_type, exc_value, tb): + """ + Allows the engine to be used as a context manager; simply stops the engine + """ + self.shutdown() + +class WindowsEngine(GenericEngine): + def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) + # Instantialize a PyDivert channel, which we will use to redirect packets + self.divert = None + self.divert_thread = None + self.divert_thread_started = False + self.interface = None # Using lazy evaluating as divert should know this + + def initialize(self): + """ + Initializes Divert such that all packets for the connection will come through us + """ + + self.logger.debug("Engine created with strategy %s (ID %s) to port %s", + str(self.strategy).strip(), self.environment_id, self.server_port) + + self.logger.debug("Initializing Divert") + + self.divert = pydivert.WinDivert("tcp.DstPort == %d || tcp.SrcPort == %d" % (int(self.server_port), int(self.server_port))) + self.divert.open() + self.divert_thread = threading.Thread(target=self.run_divert) + self.divert_thread.start() + + maxwait = 100 # 100 time steps of 0.01 seconds for a max wait of 10 seconds + i = 0 + # Give Divert time to startup, since it's running in background threads + # Block the main thread until this is done + while not self.divert_thread_started and i < maxwait: + time.sleep(0.1) + i += 1 + self.logger.debug("Divert Initialized after %d", int(i)) + + return + + def shutdown(self): + """ + Closes the divert connection + """ + if self.divert: + self.divert.close() + self.divert = None + + def run_divert(self): + """ + Runs actions on packets + """ + if self.divert: + self.divert_thread_started = True + + for packet in self.divert: + if not self.interface: + self.interface = packet.interface + if packet.is_outbound: + # Send to outbound action tree, if any + self.handle_outbound_packet(packet) + + elif packet.is_inbound: + # Send to inbound action tree, if any + self.handle_inbound_packet(packet) + + def mysend(self, packet, dir): + """ + Helper scapy sending method. Expects a Geneva Packet input. + """ + try: + self.logger.debug("Sending packet %s", str(packet)) + # Convert the packet to a bytearray so memoryview can edit the underlying memory + pack = bytearray(bytes(packet.packet)) + # Don't recalculate checksum since sometimes we will have already changed it + self.divert.send(pydivert.Packet(memoryview(pack), self.interface, dir), recalculate_checksum=False) + except Exception: + self.logger.exception("Error in engine mysend.") + + def handle_outbound_packet(self, divert_packet): + """ + Handles outbound packets by sending them the the strategy + """ + packet = actions.packet.Packet(IP(divert_packet.raw.tobytes())) + self.logger.debug("Received outbound packet %s", str(packet)) + + # Record this packet for a .pcap later + self.seen_packets.append(packet) + + packets_to_send = self.strategy.act_on_packet(packet, self.logger, direction="out") + + # Send all of the packets we've collected to send + for out_packet in packets_to_send: + self.mysend(out_packet, Direction.OUTBOUND) + + def handle_inbound_packet(self, divert_packet): + """ + Handles inbound packets. Process the packet and forward it to the strategy if needed. + """ + + packet = actions.packet.Packet(IP(divert_packet.raw.tobytes())) + + self.seen_packets.append(packet) + + self.logger.debug("Received packet: %s", str(packet)) + + # Run the given strategy + packets = self.strategy.act_on_packet(packet, self.logger, direction="in") + + # GFW will send RA packets to disrupt a TCP stream + if packet.haslayer("TCP") and packet.get("TCP", "flags") == "RA": + self.logger.debug("Detected GFW censorship - strategy failed.") + self.censorship_detected = True + + # Branching is disabled for the in direction, so we can only ever get + # back 1 or 0 packets. If zero, return and do not send packet. + if not packets: + return + + # If the strategy requested us to sleep before accepting on this packet, do so here + if packets[0].sleep: + time.sleep(packets[0].sleep) + + # Accept the modified packet + self.mysend(packets[0], Direction.INBOUND) + +class LinuxEngine(GenericEngine): + def __init__(self, server_port, string_strategy, environment_id=None, output_directory="trials", log_level="info"): + super().__init__(server_port, string_strategy, environment_id=environment_id, output_directory=output_directory, log_level=log_level) # Setup variables used by the NFQueue system self.out_nfqueue_started = False self.in_nfqueue_started = False @@ -60,26 +243,17 @@ class Engine(): self.in_nfqueue_socket = None self.out_nfqueue_thread = None self.in_nfqueue_thread = None - self.censorship_detected = False + # Specifically define an L3Socket to send our packets. This is an optimization # for scapy to send packets more quickly than using just send(), as under the hood # send() creates and then destroys a socket each time, imparting a large amount # of overhead. self.socket = conf.L3socket(iface=actions.utils.get_interface()) - def __enter__(self): - """ - Allows the engine to be used as a context manager; simply launches the - engine. - """ - self.initialize_nfqueue() - return self - - def __exit__(self, exc_type, exc_value, tb): """ Allows the engine to be used as a context manager; simply stops the engine """ - self.shutdown_nfqueue() + self.shutdown() def mysend(self, packet): """ @@ -149,7 +323,7 @@ class Engine(): subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) return cmds - def initialize_nfqueue(self): + def initialize(self): """ Initializes the nfqueue for input and output forests. """ @@ -192,7 +366,7 @@ class Engine(): i += 1 self.logger.debug("NFQueue Initialized after %d", int(i)) - def shutdown_nfqueue(self): + def shutdown(self): """ Shutdown nfqueue. """ @@ -296,7 +470,6 @@ class Engine(): # Accept the modified packet nfpacket.accept() - def get_args(): """ Sets up argparse and collects arguments. @@ -305,6 +478,7 @@ def get_args(): parser.add_argument('--server-port', type=int, action='store', required=True) parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test. If not provided, one will be generated.") parser.add_argument('--strategy', action='store', help="Strategy to deploy") + parser.add_argument('--strategy-index', action='store', help="Strategy to deploy, specified by index in the library") parser.add_argument('--output-directory', default="trials", action='store', help="Where to output logs, captures, and results. Defaults to trials/.") parser.add_argument('--log', action='store', default="debug", choices=("debug", "info", "warning", "critical", "error"), @@ -313,23 +487,31 @@ def get_args(): args = parser.parse_args() return args - def main(args): """ Kicks off the engine with the given arguments. """ + try: + if args["strategy"]: + strategy = args["strategy"] + elif args["strategy_index"]: + strategy = LIBRARY[int(args["strategy_index"])][0] + else: + # Default to first strategy + strategy = LIBRARY[0][0] eng = Engine(args["server_port"], - args["strategy"], - environment_id=args.get("environment_id"), - output_directory = args.get("output_directory"), - log_level=args["log"]) - eng.initialize_nfqueue() + strategy, + environment_id=args.get("environment_id"), + output_directory = args.get("output_directory"), + log_level=args["log"]) + eng.initialize() while True: time.sleep(0.5) + except Exception as e: + print(e) finally: - eng.shutdown_nfqueue() - + eng.shutdown() if __name__ == "__main__": main(vars(get_args())) diff --git a/library.py b/library.py new file mode 100644 index 0000000..b95d6c3 --- /dev/null +++ b/library.py @@ -0,0 +1,26 @@ +LIBRARY = [ + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-|", 94, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-|", 80, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8},),)-|", 98, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt},),)-|", 87, 100, 0), + ("[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|", 3, 100, 0), + ("[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|", 3, 0, 100), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-|", 95, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-|", 87, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 86, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:corrupt},))-|", 80, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-|", 94, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-|", 94, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))-|", 89, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-|", 96, 0, 0), + ("[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-|", 94, 0, 0), + ("[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:corrupt}-|", 94, 100, 100), + ("[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-|", 98, 100, 100), + ("[TCP:flags:PA]-fragment{tcp:-1:True}-|", 3, 100, 100), + ("[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:78},),)-|", 53, 0, 100), + ("[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|", 3, 100, 0), + ("[TCP:flags:PA]-tamper{TCP:options-uto:corrupt}-|", 3, 0, 100) +] \ No newline at end of file diff --git a/requirements.txt b/requirements_linux.txt similarity index 90% rename from requirements.txt rename to requirements_linux.txt index 42fc3fe..0840c7a 100644 --- a/requirements.txt +++ b/requirements_linux.txt @@ -4,4 +4,4 @@ netifaces netfilterqueue cryptography==2.5 requests -anytree +anytree \ No newline at end of file diff --git a/requirements_windows.txt b/requirements_windows.txt new file mode 100644 index 0000000..3f2872a --- /dev/null +++ b/requirements_windows.txt @@ -0,0 +1,7 @@ +scapy==2.4.3 +requests +netifaces +cryptography==2.5 +requests +anytree +pydivert \ No newline at end of file diff --git a/tests/test_fragment.py b/tests/test_fragment.py index 3e7bcee..03a1c64 100644 --- a/tests/test_fragment.py +++ b/tests/test_fragment.py @@ -217,4 +217,86 @@ def test_ip_only_fragment(): assert packet1["Raw"].load == b'datadata', "Left packet incorrectly fragmented" assert packet2["Raw"].load == b"11datadata", "Right packet incorrectly fragmented" +def test_overlapping_segment(): + """ + Basic test for overlapping segments. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:4}", logger) + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11dat', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_no_overlap(): + """ + Basic test for overlapping segments with no overlap. (shouldn't ever actually happen) + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:0}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata1', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_entire_packet(): + """ + Basic test for overlapping segments overlapping entire packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:9}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_out_of_bounds(): + """ + Basic test for overlapping segments overlapping beyond the edge of the packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:20}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segmentation_parse(): + """ + Basic test for parsing overlapping segments. + """ + + fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=2, segment=True, overlap=3) + assert str(fragment) == "fragment{tcp:2:False:3}", "Fragment returned incorrect string representation: %s" % str(fragment)