mirror of https://github.com/Kkevsterrr/geneva
Merge pull request #15 from Kkevsterrr/GEN-1_genetic_algorithm
Added ReadTheDocs documentation & genetic algorithm
This commit is contained in:
commit
293651a598
|
@ -9,8 +9,12 @@ exclude_lines =
|
|||
raise NotImplementedError
|
||||
if __name__ == .__main__.:
|
||||
def get_args
|
||||
def main
|
||||
except KeyboardInterrupt:
|
||||
ignore_errors = True
|
||||
omit =
|
||||
experimental/*
|
||||
experimental/smtp/*
|
||||
tests/*
|
||||
examples/*
|
||||
censors/*
|
||||
censor_driver.py
|
||||
graph.py
|
||||
|
|
35
.travis.yml
35
.travis.yml
|
@ -1,39 +1,32 @@
|
|||
sudo: required
|
||||
dist: "bionic"
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
language: python
|
||||
|
||||
python:
|
||||
- "3.6"
|
||||
|
||||
install:
|
||||
# Travis recently added systemd-resolvd to their VMs. Since full Geneva often runs its own DNS
|
||||
# server to test DNS strategies, we need to disable system-resolvd.
|
||||
# First disable the service
|
||||
- sudo systemctl disable systemd-resolved.service
|
||||
# Stop the service
|
||||
- sudo systemctl stop systemd-resolved
|
||||
# With systemd not running, our own hostname won't resolve - this causes issues with sudo.
|
||||
# Add back our hostname to /etc/hosts/ so sudo does not complain
|
||||
- echo $(hostname -I | cut -d\ -f1) $(hostname) | sudo tee -a /etc/hosts
|
||||
# Replace the 127.0.0.53 nameserver with Google's
|
||||
- sudo sed 's/nameserver.*/nameserver 8.8.8.8/' /etc/resolv.conf > /tmp/resolv.conf.new
|
||||
# Travis recently added systemd-resolvd to their VMs. We must disable this to test our own DNS server
|
||||
- sudo systemctl disable systemd-resolved.service # First disable the service
|
||||
- sudo systemctl stop systemd-resolved # Stop the service
|
||||
- echo $(hostname -I | cut -d\ -f1) $(hostname) | sudo tee -a /etc/hosts # add back our hostname to /etc/hosts so sudo does not complain
|
||||
- sudo sed 's/nameserver.*/nameserver 8.8.8.8/' /etc/resolv.conf > /tmp/resolv.conf.new # replace the 127.0.0.53 nameserver with Google's
|
||||
- sudo mv /tmp/resolv.conf.new /etc/resolv.conf
|
||||
# Now that systemd-resolv.conf is safely disabled, we can now setup for Geneva
|
||||
- sudo systemctl start docker # must now restart docker so the resolv.conf changes propagate to its containers
|
||||
- sudo apt-get clean # travis having mirror sync issues
|
||||
# Install dependencies
|
||||
- sudo apt-get update
|
||||
- sudo apt-get -y install libnetfilter-queue-dev python3 python3-pip python3-setuptools graphviz
|
||||
# Since sudo is required but travis does not set up the root environment, we must override the
|
||||
# secure_path in sudoers in order for travis's setup to take effect for sudo commands
|
||||
- printf "Defaults\tenv_reset\nDefaults\tmail_badpass\nDefaults\tsecure_path="/home/travis/virtualenv/python3.6.7/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"\nroot\tALL=(ALL:ALL) ALL\n#includedir /etc/sudoers.d\n" > /tmp/sudoers.tmp
|
||||
# Verify the sudoers file
|
||||
- sudo visudo -c -f /tmp/sudoers.tmp
|
||||
# Copy in the sudoers file
|
||||
- sudo cp /tmp/sudoers.tmp /etc/sudoers
|
||||
# Now that sudo is good to go, finish installing dependencies
|
||||
- printf "Defaults\tenv_reset\nDefaults\tmail_badpass\nDefaults\tsecure_path="/home/travis/virtualenv/python3.6.7/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"\nroot\tALL=(ALL:ALL) ALL\n#includedir /etc/sudoers.d\n" > /tmp/sudoers.tmp # Since sudo is required but travis does not set up the root environment, we must override the secure_path in sudoers in order for travis's setup to take effect for sudo commands
|
||||
- sudo visudo -c -f /tmp/sudoers.tmp # Verify the sudoers file
|
||||
- sudo cp /tmp/sudoers.tmp /etc/sudoers # Copy in the sudoers file
|
||||
- sudo echo $PATH # Confirm the PATH changes took effect
|
||||
- sudo python3 -m pip install -r requirements.txt
|
||||
- sudo python3 -m pip install slackclient pytest-cov
|
||||
- docker build -t base:latest -f docker/Dockerfile .
|
||||
|
||||
script:
|
||||
- sudo python3 -m pytest --cov=./ -sv tests/ --tb=short
|
||||
|
|
133
README.md
133
README.md
|
@ -1,15 +1,18 @@
|
|||
# Geneva [![Build Status](https://travis-ci.com/Kkevsterrr/geneva.svg?branch=master)](https://travis-ci.com/Kkevsterrr/geneva) [![codecov](https://codecov.io/gh/Kkevsterrr/geneva/branch/master/graph/badge.svg)](https://codecov.io/gh/Kkevsterrr/geneva)
|
||||
# Geneva [![Build Status](https://travis-ci.com/Kkevsterrr/geneva.svg?branch=master)](https://travis-ci.com/Kkevsterrr/geneva) [![codecov](https://codecov.io/gh/Kkevsterrr/geneva/branch/master/graph/badge.svg)](https://codecov.io/gh/Kkevsterrr/geneva)[![Documentation Status](https://readthedocs.org/projects/geneva/badge/?version=latest)](https://geneva.readthedocs.io/en/latest/?badge=latest)
|
||||
|
||||
Geneva is an artificial intelligence tool that defeats censorship by exploiting bugs in censors, such as those in China, India, and Kazakhstan. Unlike many other anti-censorship solutions which require assistance from outside the censoring regime (Tor, VPNs, etc.), Geneva runs strictly on the client.
|
||||
Geneva is an artificial intelligence tool that defeats censorship by exploiting bugs in censors, such as those in China, India, and Kazakhstan. Unlike many other anti-censorship solutions which require assistance from outside the censoring regime (Tor, VPNs, etc.), Geneva runs strictly on one side of the connection (either the client or server side).
|
||||
|
||||
Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion strategies and has found several previously unknown bugs in censors. Geneva's strategies manipulate the client's packets to confuse the censor without impacting the client/server communication. This makes Geneva effective against many types of in-network censorship (though it cannot be used against IP-blocking censorship).
|
||||
Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion strategies and has found several previously unknown bugs in censors. Geneva's strategies manipulate the network stream to confuse the censor without impacting the client/server communication. This makes Geneva effective against many types of in-network censorship (though it cannot be used against IP-blocking censorship).
|
||||
|
||||
This code release specifically contains the strategy engine used by Geneva, its Python API, and a subset of published strategies, so users and researchers can test and deploy Geneva's strategies. To learn more about how Geneva works, visit [How it Works](#How-it-Works). We will be releasing the genetic algorithm at a later date.
|
||||
Geneva is composed of two high level components: its genetic algorithm (which it uses to evolve new censorship evasion strategies) and its strategy engine (which is uses to run an individual censorship evasion strategy over a network connection).
|
||||
|
||||
This codebase contains the Geneva's full implementation: its genetic algorithm, strategy engine, Python API, and a subset of published strategies. With these tools, users and researchers alike can evolve new strategies or leverage existing strategies to evade censorship. To learn more about how Geneva works, see [How it Works](#How-it-Works) or checkout our [documentation](https://geneva.readthedocs.io).
|
||||
|
||||
## Setup
|
||||
|
||||
Geneva has been developed and tested for Centos or Debian-based systems. Due to limitations of
|
||||
netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6* (with more versions coming soon).
|
||||
netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6*.
|
||||
More detailed setup instructions are available at our [documentation](https://geneva.readthedocs.io).
|
||||
|
||||
Install netfilterqueue dependencies:
|
||||
```
|
||||
|
@ -21,7 +24,10 @@ Install Python dependencies:
|
|||
# python3 -m pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Running it
|
||||
## Running a Strategy
|
||||
|
||||
A censorship evasion strategy is simply a _description of how network traffic should be modified_. A strategy is not
|
||||
code, it is a description that tells the engine how it should operate over traffic. For a fuller description of the DNA syntax, see [Censorship Evasion Strategies](#Censorship-Evasion-Strategies).
|
||||
|
||||
```
|
||||
# python3 engine.py --server-port 80 --strategy "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|" --log debug
|
||||
|
@ -38,27 +44,67 @@ this will fail. To fix this, remove those rules.
|
|||
|
||||
## Strategy Library
|
||||
|
||||
Geneva has found dozens of strategies that work against censors in China, Kazakhstan, and India. We include several of these strategies in [strategies.md](strategies.md). Note that this file contains success rates for each individual country; a strategy that works in one country may not work as well as other countries.
|
||||
Geneva has found dozens of strategies that work against censors in China, Kazakhstan, India, and Iran. We include several of these strategies in [strategies.md](strategies.md). Note that this file contains success rates for each individual country; a strategy that works in one country may not work as well as other countries.
|
||||
|
||||
Researchers have observed that strategies may have differing success rates based on your exact location. Although we have not observed this from our vantage points, you may find that some strategies may work differently in a country we have tested. If this is the case, don't be alarmed. However, please feel free to reach out to a member of the team directly or open an issue on this page so we can track how the strategies work from other geographic locations.
|
||||
|
||||
## Disclaimer
|
||||
|
||||
Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it.
|
||||
Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it.
|
||||
|
||||
-------
|
||||
|
||||
## How it Works
|
||||
# How it Works
|
||||
|
||||
See our paper for an in-depth read on how Geneva works. Below is a rundown of the format of Geneva's strategy DNA.
|
||||
See our [paper](#Paper) for an in-depth read on how Geneva works. Below is a walkthrough of the main concepts behind Geneva, the major components of the codebase, and how they can be used.
|
||||
|
||||
## Censorship Evasion Strategies
|
||||
|
||||
A censorship evasion strategy is simply a _description of how network traffic should be modified_. A strategy is _not
|
||||
code_, it is a description that tells Geneva's stratgy engine how it should manipulate network traffic. The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted.
|
||||
|
||||
A censorship evasion strategy composed of one or more packet-level building blocks. Geneva's core building blocks are:
|
||||
1. `duplicate`: takes one packet and returns two copies of the packet
|
||||
2. `drop`: takes one packet and returns no packets (drops the packet)
|
||||
3. `tamper`: takes one packet and returns the modified packet
|
||||
4. `fragment`: takes one packet and returns two fragments or two segments
|
||||
|
||||
Since `duplicate` and `fragment` introduce _branching_, these actions are composed into a binary-tree structure called an _action tree_. Each tree also has a _trigger_. The trigger describes which packets the tree should run on, and the tree describes what should happen to each of those packets when the trigger fires. Once a trigger fires on a packet, it pulls the packet into the tree for modifications, and the packets that emerge from the tree are sent on the wire. Recall that Geneva operates at the packet level, therefore all triggers are packet-level triggers.
|
||||
|
||||
Multiple action trees together form a _forest_. Geneva handles outbound and inbound packets differently, so strategies are composed of two forests: an outbound forest and an inbound forest.
|
||||
|
||||
Consider the following example of a simple Geneva strategy.
|
||||
```
|
||||
+---------------+
|
||||
| TCP:flags:A | <-- triggers on TCP packets with the flags field set to 'ACK'
|
||||
+-------+-------+ matching packets are captured and pulled into the tree
|
||||
|
|
||||
+---------v---------+
|
||||
duplicate <-- makes two copies of the given packet. the tree is processed
|
||||
+---------+---------+ with an inorder traversal, so the left side is run first
|
||||
|
|
||||
+-------------+------------+
|
||||
| |
|
||||
+------------v----------+ v <-- dupilcate has no right child, so this packet will be sent on the wire unimpacted
|
||||
tamper
|
||||
{TCP:flags:replace:R} <-- parameters to this action describe how the packet should be tampered
|
||||
+------------+----------+
|
||||
|
|
||||
+------------v----------+
|
||||
tamper
|
||||
{TCP:chksum:corrupt}
|
||||
+------------+----------+
|
||||
|
|
||||
v <-- packets that emerge from an in-order traversal of the leaves are sent on the wire
|
||||
|
||||
```
|
||||
|
||||
This strategy triggers on `TCP` packets with the `flags` field set to `ACK`. It makes a duplicate of the `ACK` packet; the first duplicate has its flags field changed to `RST` and its checksum (`chksum`) field corrupted; the second duplicate is unchaged. Both packets are then sent on the network.
|
||||
|
||||
### Strategy DNA
|
||||
|
||||
Geneva's strategies can be arbitrarily complicated, and it defines a well-formatted syntax for
|
||||
expressing strategies to the engine.
|
||||
|
||||
A strategy is simply a _description of how network traffic should be modified_. A strategy is not
|
||||
code, it is a description that tells the engine how it should operate over traffic.
|
||||
These strategies can be arbitrarily complicated, and Geneva defines a well-formatted string syntax for
|
||||
unambiguously expressing strategies.
|
||||
|
||||
A strategy divides how it handles outbound and inbound packets: these are separated in the DNA by a
|
||||
"\\/". Specifically, the strategy format is `<outbound forest> \/ <inbound forest>`. If `\/` is not
|
||||
|
@ -66,35 +112,50 @@ present in a strategy, all of the action trees are in the outbound forest.
|
|||
|
||||
Both forests are composed of action trees, and each forest is allowed an arbitrarily many trees.
|
||||
|
||||
An action tree is comprised of a _trigger_ and a _tree_. The trigger describes _when_ the strategy
|
||||
should run, and the tree describes what should happen when the trigger fires. Recall that Geneva
|
||||
operates at the packet level, therefore all triggers are packet-level triggers. Action trees start
|
||||
with a trigger, and always end with a `-|`.
|
||||
Action trees always start with a trigger, which is formatted as: `[<protocol>:<field>:<value>]`. For example, the trigger: `[TCP:flags:S]` will run its corresponding tree whenever it sees a `TCP` packet with the `flags` field set to `SYN`. If the corresponding action tree is `[TCP:flags:S]-drop-|`, this action tree will cause the engine to drop any `SYN` packets. `[TCP:flags:S]-duplicate-|` will cause the engine to duplicate any SYN packets.
|
||||
|
||||
Triggers operate as exact-matches, are formatted as follows: `[<protocol>:<field>:<value>]`. For
|
||||
example, the trigger: `[TCP:flags:S]` will run its corresponding tree whenever it sees a `SYN`
|
||||
TCP packet. If the corresponding action tree is `[TCP:flags:S]-drop-|`, this action tree will cause
|
||||
the engine to drop any `SYN` packets. `[TCP:flags:S]-duplicate-|` will cause the engine to
|
||||
duplicate the SYN packet.
|
||||
Syntactically, action trees end with `-|`.
|
||||
|
||||
Depending on the type of action, some actions can have up to two children. These are represented
|
||||
Depending on the type of action, some actions can have up to two children (such as `duplicate`). These are represented
|
||||
with the following syntax: `[TCP:flags:S]-duplicate(<left_child>,<right_child>)-|`, where
|
||||
`<left_child>` and `<right_child>` themselves are trees. If `(,)` is not specified, any packets
|
||||
that emerge from the action will be sent on the wire.
|
||||
that emerge from the action will be sent on the wire. If an action only has one child (such as `tamper`), it is always the left child. `[TCP:flags:S]-tamper{<parameters>}(<left_child>,)-|`
|
||||
|
||||
Any action that has parameters associated with it contain those parameters in `{}`. Consider the
|
||||
following strategy with `tamper`.
|
||||
```
|
||||
[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R},)-| \/
|
||||
```
|
||||
This strategy takes outbound `ACK` packets and duplicates them. To the first duplicate, it tampers
|
||||
the packet by replacing the `TCP` `flags` field with `RST`, and does nothing to the second
|
||||
duplicate.
|
||||
Actions that have parameters specify those parameters within `{}`. For example, giving parameters to the `tamper` action could look like: `[TCP:flags:S]-tamper{TCP:flags:replace:A}-|`. This strategy would trigger on TCP `SYN` packets and replace the TCP `flags` field to `ACK`.
|
||||
|
||||
Note that due to NFQueue limitations, actions that introduce branching (fragment, duplicate) are
|
||||
Putting this all together, below is the strategy DNA representation of the above diagram:
|
||||
```
|
||||
[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/
|
||||
```
|
||||
|
||||
Geneva has code to parse this strategy DNA into strategies that can be applied to network traffic using the engine.
|
||||
|
||||
Note that due to limitations of Scapy and NFQueue, actions that introduce branching (`fragment`, `duplicate`) are
|
||||
disabled for incoming action forests.
|
||||
|
||||
-------
|
||||
## Engine
|
||||
|
||||
The strategy engine (`engine.py`) applies a strategy to a network connection. The engine works by capturing all traffic to/from a specified port. Packets that match an active trigger are run through the associated action-tree, and packets that emerge from the tree are sent on the wire.
|
||||
|
||||
The engine also has a Python API for using it in your application. It can be used as a context manager or invoked in the background as a thread.
|
||||
For example, consider the following simple application.
|
||||
|
||||
```python
|
||||
import os
|
||||
import engine
|
||||
|
||||
# Port to run the engine on
|
||||
port = 80
|
||||
# Strategy to use
|
||||
strategy = "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/"
|
||||
|
||||
# Create the engine in debug mode
|
||||
with engine.Engine(port, strategy, log_level="debug") as eng:
|
||||
os.system("curl http://example.com?q=ultrasurf")
|
||||
```
|
||||
This script creates an instance of the engine with a specified strategy, and that strategy will be running for everything within the context manager. When the context manager exits, the engine will clean itself up. See the `examples/` folder for more use cases of the engine.
|
||||
|
||||
Due to limitations of scapy and NFQueue, the engine cannot be used to communicate with localhost.
|
||||
|
||||
## Citation
|
||||
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
"""
|
||||
Action
|
||||
|
||||
Geneva object for defining a packet-level action.
|
||||
Geneva superclass object for defining a packet-level action.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
@ -17,16 +15,24 @@ ACTION_CACHE["in"] = {}
|
|||
ACTION_CACHE["out"] = {}
|
||||
BASEPATH = os.path.sep.join(os.path.dirname(os.path.abspath(__file__)).split(os.path.sep)[:-1])
|
||||
|
||||
|
||||
class Action():
|
||||
"""
|
||||
Defines the superclass for a Geneva Action.
|
||||
"""
|
||||
# Give each Action a unique ID - this is needed for graphing/visualization
|
||||
ident = 0
|
||||
# Each Action has a 'frequency' field - this defines how likely it is to be chosen
|
||||
# when a new action is chosen
|
||||
frequency = 0
|
||||
|
||||
def __init__(self, action_name, direction):
|
||||
"""
|
||||
Initializes this action object.
|
||||
|
||||
Args:
|
||||
action_name (str): Name of this action ("duplicate")
|
||||
direction (str): Direction of this action ("out", "both", "in")
|
||||
"""
|
||||
self.enabled = True
|
||||
self.action_name = action_name
|
||||
|
@ -45,6 +51,12 @@ class Action():
|
|||
"""
|
||||
Returns whether this action applies to the given direction, as
|
||||
branching actions are not supported on inbound trees.
|
||||
|
||||
Args:
|
||||
direction (str): Direction to check if this action applies ("out", "in", "both")
|
||||
|
||||
Returns:
|
||||
bool: whether or not this action can be used to a given direction
|
||||
"""
|
||||
if direction == self.direction or self.direction == "both":
|
||||
return True
|
||||
|
@ -67,6 +79,14 @@ class Action():
|
|||
Dynamically imports all of the Action classes in this directory.
|
||||
|
||||
Will only return terminal actions if terminal is set to True.
|
||||
|
||||
Args:
|
||||
direction (str): Limit imported actions to just those that can run to this direction ("out", "in", "both")
|
||||
disabled (list, optional): list of actions that are disabled
|
||||
allow_terminal (bool): whether or not terminal actions ("drop") should be imported
|
||||
|
||||
Returns:
|
||||
dict: Dictionary of imported actions
|
||||
"""
|
||||
if disabled is None:
|
||||
disabled = []
|
||||
|
@ -84,7 +104,6 @@ class Action():
|
|||
else:
|
||||
return ACTION_CACHE[direction][terminal]
|
||||
|
||||
|
||||
collected_actions = []
|
||||
# Get the base path for the project relative to this file
|
||||
path = os.path.join(BASEPATH, "actions")
|
||||
|
@ -117,6 +136,14 @@ class Action():
|
|||
def parse_action(str_action, direction, logger):
|
||||
"""
|
||||
Parses a string action into the action object.
|
||||
|
||||
Args:
|
||||
str_action (str): String representation of an action to parse
|
||||
direction (str): Limit actions searched through to just those that can run to this direction ("out", "in", "both")
|
||||
logger (:obj:`logging.Logger`): a logger to log with
|
||||
|
||||
Returns:
|
||||
:obj:`action.Action`: A parsed action object
|
||||
"""
|
||||
# Collect all viable actions that can run for each respective direction
|
||||
outs = Action.get_actions("out")
|
||||
|
|
|
@ -1,7 +1,18 @@
|
|||
from actions.action import Action
|
||||
|
||||
class DropAction(Action):
|
||||
"""
|
||||
Geneva action to drop the given packet.
|
||||
"""
|
||||
frequency = 1
|
||||
|
||||
def __init__(self, environment_id=None):
|
||||
"""
|
||||
Initializes this drop action.
|
||||
|
||||
Args:
|
||||
environment_id (str, optional): Environment ID of the strategy we are a part of
|
||||
"""
|
||||
Action.__init__(self, "drop", "both")
|
||||
self.terminal = True
|
||||
self.branching = False
|
||||
|
|
|
@ -2,6 +2,10 @@ from actions.action import Action
|
|||
|
||||
|
||||
class DuplicateAction(Action):
|
||||
"""
|
||||
Defines the DuplicateAction - returns two copies of the given packet.
|
||||
"""
|
||||
frequency = 3
|
||||
def __init__(self, environment_id=None):
|
||||
Action.__init__(self, "duplicate", "out")
|
||||
self.branching = True
|
||||
|
@ -13,3 +17,9 @@ class DuplicateAction(Action):
|
|||
"""
|
||||
logger.debug(" - Duplicating given packet %s" % str(packet))
|
||||
return packet, packet.copy()
|
||||
|
||||
def mutate(self, environment_id=None):
|
||||
"""
|
||||
Swaps its left and right child
|
||||
"""
|
||||
self.left, self.right = self.right, self.left
|
||||
|
|
|
@ -1,23 +1,36 @@
|
|||
import random
|
||||
from actions.action import Action
|
||||
import actions.packet
|
||||
import layers.packet
|
||||
|
||||
from scapy.all import IP, TCP, fragment
|
||||
|
||||
|
||||
MAX_UINT = 4294967295
|
||||
|
||||
|
||||
class FragmentAction(Action):
|
||||
def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True):
|
||||
'''
|
||||
correct_order specifies if the fragmented packets should come in the correct order
|
||||
fragsize specifies how
|
||||
'''
|
||||
"""
|
||||
Defines the FragmentAction for Geneva - fragments or segments the given packet.
|
||||
"""
|
||||
frequency = 2
|
||||
def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True, overlap=0):
|
||||
"""
|
||||
Initializes a fragment action object.
|
||||
|
||||
Args:
|
||||
environment_id (str, optional): Environment ID of the strategy this object is a part of
|
||||
correct_order (bool, optional): Whether or not the fragments/segments should be returned in the correct order
|
||||
fragsize (int, optional): The index this packet should be cut. Defaults to -1, which cuts it in half.
|
||||
segment (bool, optional): Whether we should perform fragmentation or segmentation
|
||||
overlap (int, optional): How many bytes the fragments/segments should overlap
|
||||
"""
|
||||
Action.__init__(self, "fragment", "out")
|
||||
self.enabled = True
|
||||
self.branching = True
|
||||
self.terminal = False
|
||||
self.fragsize = fragsize
|
||||
self.segment = segment
|
||||
|
||||
self.overlap = overlap
|
||||
if correct_order == None:
|
||||
self.correct_order = self.get_rand_order()
|
||||
else:
|
||||
|
@ -75,8 +88,8 @@ class FragmentAction(Action):
|
|||
else:
|
||||
# packet can be fragmented as requested
|
||||
frags = self.fragment(packet.copy().packet, fragsize=self.fragsize*8)
|
||||
packet1 = actions.packet.Packet(frags[0])
|
||||
packet2 = actions.packet.Packet(frags[1])
|
||||
packet1 = layers.packet.Packet(frags[0])
|
||||
packet2 = layers.packet.Packet(frags[1])
|
||||
if self.correct_order:
|
||||
return packet1, packet2
|
||||
else:
|
||||
|
@ -87,6 +100,9 @@ class FragmentAction(Action):
|
|||
Segments a packet into two, given the size of the first packet (0:fragsize)
|
||||
Always returns two packets, since fragment is a branching action, so if we
|
||||
are unable to segment, it will duplicate the packet.
|
||||
|
||||
If overlap is specified, it will select n bytes from the second packet
|
||||
and append them to the first, and increment the sequence number accordingly
|
||||
"""
|
||||
if not packet.haslayer("TCP") or not hasattr(packet["TCP"], "load") or not packet["TCP"].load:
|
||||
return packet, packet.copy() # duplicate if no TCP or no payload to segment
|
||||
|
@ -101,7 +117,11 @@ class FragmentAction(Action):
|
|||
fragsize = int(len(payload)/2)
|
||||
|
||||
# Craft new packets
|
||||
pkt1 = IP(packet["IP"])/payload[:fragsize]
|
||||
|
||||
# Make sure we don't go out of bounds by choosing the min
|
||||
overlap_bytes = min(len(payload[fragsize:]), self.overlap)
|
||||
# Attach these bytes to the first packet
|
||||
pkt1 = IP(packet["IP"])/payload[:fragsize + overlap_bytes]
|
||||
pkt2 = IP(packet["IP"])/payload[fragsize:]
|
||||
|
||||
# We cannot rely on scapy's native parsing here - if a previous action has changed the
|
||||
|
@ -112,10 +132,14 @@ class FragmentAction(Action):
|
|||
if not pkt2.haslayer("TCP"):
|
||||
pkt2 = IP(packet["IP"])/TCP(bytes(pkt2["IP"].load))
|
||||
|
||||
packet1 = actions.packet.Packet(pkt1)
|
||||
packet2 = actions.packet.Packet(pkt2)
|
||||
packet1 = layers.packet.Packet(pkt1)
|
||||
packet2 = layers.packet.Packet(pkt2)
|
||||
|
||||
# Reset packet2's SYN number
|
||||
if packet2["TCP"].seq + fragsize > MAX_UINT:
|
||||
# Wrap sequence numbers around if greater than MAX_UINT
|
||||
packet2["TCP"].seq = packet2["TCP"].seq + fragsize - MAX_UINT - 1
|
||||
else:
|
||||
packet2["TCP"].seq += fragsize
|
||||
|
||||
del packet1["IP"].chksum
|
||||
|
@ -147,10 +171,14 @@ class FragmentAction(Action):
|
|||
Returns a string representation with the fragsize
|
||||
"""
|
||||
s = Action.__str__(self)
|
||||
if self.segment:
|
||||
s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + "}"
|
||||
if not self.overlap:
|
||||
ending = "}"
|
||||
else:
|
||||
s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + "}"
|
||||
ending = ":" + str(self.overlap) + "}"
|
||||
if self.segment:
|
||||
s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + ending
|
||||
else:
|
||||
s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + ending
|
||||
return s
|
||||
|
||||
def parse(self, string, logger):
|
||||
|
@ -169,22 +197,36 @@ class FragmentAction(Action):
|
|||
num_parameters = string.count(":")
|
||||
|
||||
# If num_parameters is greater than 2, it's not a valid fragment action
|
||||
if num_parameters != 2:
|
||||
msg = "Cannot parse fragment action %s" % string
|
||||
logger.error(msg)
|
||||
raise Exception(msg)
|
||||
else:
|
||||
if num_parameters == 2:
|
||||
params = string.split(":")
|
||||
seg, fragsize, correct_order = params
|
||||
overlap = 0
|
||||
if "tcp" in seg:
|
||||
self.segment = True
|
||||
else:
|
||||
self.segment = False
|
||||
|
||||
elif num_parameters == 3:
|
||||
params = string.split(":")
|
||||
seg, fragsize, correct_order, overlap = params
|
||||
if overlap.endswith("}"):
|
||||
overlap = overlap[:-1] # Chop off trailing }
|
||||
if "tcp" in seg:
|
||||
self.segment = True
|
||||
else:
|
||||
self.segment = False
|
||||
|
||||
else:
|
||||
msg = "Cannot parse fragment action %s" % string
|
||||
logger.error(msg)
|
||||
raise Exception(msg)
|
||||
|
||||
try:
|
||||
# Try to convert to int
|
||||
self.fragsize = int(fragsize)
|
||||
except ValueError:
|
||||
self.overlap = int(overlap)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
msg = "Cannot parse fragment action %s" % string
|
||||
logger.error(msg)
|
||||
raise Exception(msg)
|
||||
|
@ -196,3 +238,33 @@ class FragmentAction(Action):
|
|||
self.correct_order = False
|
||||
|
||||
return True
|
||||
|
||||
def mutate(self, environment_id=None):
|
||||
"""
|
||||
Mutates the fragment action - it either chooses a new segment offset,
|
||||
switches the packet order, and/or changes whether it segments or fragments.
|
||||
"""
|
||||
self.correct_order = self.get_rand_order()
|
||||
self.segment = random.choice([True, True, True, False])
|
||||
if self.segment:
|
||||
if random.random() < 0.5:
|
||||
self.fragsize = int(random.uniform(1, 60))
|
||||
else:
|
||||
self.fragsize = -1
|
||||
else:
|
||||
if random.random() < 0.2:
|
||||
self.fragsize = int(random.uniform(1, 50))
|
||||
else:
|
||||
self.fragsize = -1
|
||||
|
||||
if random.random() < .5:
|
||||
# Somewhat aggressively overlap
|
||||
if random.random() < .5:
|
||||
if self.fragsize == -1:
|
||||
self.overlap = 5
|
||||
else:
|
||||
self.overlap = int(self.fragsize/2)
|
||||
else:
|
||||
self.overlap = int(random.uniform(1, 50))
|
||||
|
||||
return self
|
||||
|
|
835
actions/layer.py
835
actions/layer.py
|
@ -1,835 +0,0 @@
|
|||
import binascii
|
||||
import random
|
||||
import string
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
from scapy.all import IP, RandIP, UDP, DNS, DNSQR, Raw, TCP, fuzz
|
||||
|
||||
|
||||
class Layer():
|
||||
"""
|
||||
Base class defining a Geneva packet layer.
|
||||
"""
|
||||
protocol = None
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes this layer.
|
||||
"""
|
||||
self.layer = layer
|
||||
# No custom setter, getters, generators, or parsers are needed by default
|
||||
self.setters = {}
|
||||
self.getters = {}
|
||||
self.generators = {}
|
||||
self.parsers = {}
|
||||
|
||||
@classmethod
|
||||
def reset_restrictions(cls):
|
||||
"""
|
||||
Resets field restrictions placed on this layer.
|
||||
"""
|
||||
cls.fields = cls._fields
|
||||
|
||||
def get_next_layer(self):
|
||||
"""
|
||||
Given the current layer returns the next layer beneath us.
|
||||
"""
|
||||
if len(self.layer.layers()) == 1:
|
||||
return None
|
||||
|
||||
return self.layer[1]
|
||||
|
||||
def get_random(self):
|
||||
"""
|
||||
Retreives a random field and value.
|
||||
"""
|
||||
field = random.choice(self.fields)
|
||||
return field, self.get(field)
|
||||
|
||||
def gen_random(self):
|
||||
"""
|
||||
Generates a random field and value.
|
||||
"""
|
||||
assert self.fields, "Layer %s doesn't have any fields" % str(self)
|
||||
field = random.choice(self.fields)
|
||||
return field, self.gen(field)
|
||||
|
||||
@classmethod
|
||||
def name_matches(cls, name):
|
||||
"""
|
||||
Checks if given name matches this layer name.
|
||||
"""
|
||||
return name.upper() == cls.name.upper()
|
||||
|
||||
def get(self, field):
|
||||
"""
|
||||
Retrieves the value from a given field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.getters:
|
||||
return self.getters[field](field)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.getters:
|
||||
return self.getters[base](field)
|
||||
|
||||
return getattr(self.layer, field)
|
||||
|
||||
def set(self, packet, field, value):
|
||||
"""
|
||||
Sets the value for a given field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
base = field.split("-")[0]
|
||||
if field in self.setters:
|
||||
self.setters[field](packet, field, value)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
elif "-" in field and base in self.setters:
|
||||
self.setters[base](packet, field, value)
|
||||
else:
|
||||
setattr(self.layer, field, value)
|
||||
|
||||
# Request the packet be reparsed to confirm the value is stable
|
||||
# XXX Temporarily disabling the reconstitution check due to scapy bug (#2034)
|
||||
#assert bytes(self.protocol(bytes(self.layer))) == bytes(self.layer)
|
||||
|
||||
def gen(self, field):
|
||||
"""
|
||||
Generates a value for this field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.generators:
|
||||
return self.generators[field](field)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.generators:
|
||||
return self.generators[base](field)
|
||||
|
||||
sample = fuzz(self.protocol())
|
||||
|
||||
new_value = getattr(sample, field)
|
||||
if new_value == None:
|
||||
new_value = 0
|
||||
elif type(new_value) != int:
|
||||
new_value = new_value._fix()
|
||||
|
||||
return new_value
|
||||
|
||||
def parse(self, field, value):
|
||||
"""
|
||||
Parses the given value for a given field. This is useful for fields whose
|
||||
value cannot be represented in a string type easily - it lets us define
|
||||
a common string representation for the strategy, and parse it back into
|
||||
a real value here.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.parsers:
|
||||
return self.parsers[field](field, value)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.parsers:
|
||||
return self.parsers[base](field, value)
|
||||
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError:
|
||||
parsed = value
|
||||
|
||||
return parsed
|
||||
|
||||
def get_load(self, field):
|
||||
"""
|
||||
Helper method to retrieve load, as scapy doesn't recognize 'load' as
|
||||
a regular field properly.
|
||||
"""
|
||||
try:
|
||||
load = self.layer.payload.load
|
||||
except AttributeError:
|
||||
pass
|
||||
try:
|
||||
load = self.layer.load
|
||||
except AttributeError:
|
||||
return ""
|
||||
|
||||
if not load:
|
||||
return ""
|
||||
|
||||
return urllib.parse.quote(load.decode('utf-8', 'ignore'))
|
||||
|
||||
def set_load(self, packet, field, value):
|
||||
"""
|
||||
Helper method to retrieve load, as scapy doesn't recognize 'load' as
|
||||
a field properly.
|
||||
"""
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].len
|
||||
|
||||
value = urllib.parse.unquote(value)
|
||||
|
||||
value = value.encode('utf-8')
|
||||
# Add support for injecting arbitrary protocol payloads if requested
|
||||
dns_payload = b"\x009ib\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x08examples\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00\x01\x00\x00\x01+\x00\x04\xc7\xbf2I\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00"
|
||||
http_payload = b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
|
||||
|
||||
value = value.replace(b"__DNS_REQUEST__", dns_payload)
|
||||
value = value.replace(b"__HTTP_REQUEST__", http_payload)
|
||||
|
||||
self.layer.payload = Raw(value)
|
||||
|
||||
def gen_load(self, field):
|
||||
"""
|
||||
Helper method to generate a random load, as scapy doesn't recognize 'load'
|
||||
as a field properly.
|
||||
"""
|
||||
load = ''.join([random.choice(string.ascii_lowercase + string.digits) for k in range(10)])
|
||||
return urllib.parse.quote(load)
|
||||
|
||||
|
||||
class IPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access IP header fields.
|
||||
"""
|
||||
name = "IP"
|
||||
protocol = IP
|
||||
_fields = [
|
||||
'version',
|
||||
'ihl',
|
||||
'tos',
|
||||
'len',
|
||||
'id',
|
||||
'flags',
|
||||
'frag',
|
||||
'ttl',
|
||||
'proto',
|
||||
'chksum',
|
||||
'src',
|
||||
'dst',
|
||||
'load'
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the IP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
"flags" : self.get_flags,
|
||||
"load" : self.get_load
|
||||
}
|
||||
self.setters = {
|
||||
"flags" : self.set_flags,
|
||||
"load" : self.set_load
|
||||
}
|
||||
self.generators = {
|
||||
"src" : self.gen_ip,
|
||||
"dst" : self.gen_ip,
|
||||
"chksum" : self.gen_chksum,
|
||||
"len" : self.gen_len,
|
||||
"load" : self.gen_load,
|
||||
"flags" : self.gen_flags
|
||||
}
|
||||
|
||||
def gen_len(self, field):
|
||||
"""
|
||||
Generates a valid IP length. Scapy breaks if the length is set to 0, so
|
||||
return a random int starting at 1.
|
||||
"""
|
||||
return random.randint(1, 500)
|
||||
|
||||
def gen_chksum(self, field):
|
||||
"""
|
||||
Generates a checksum.
|
||||
"""
|
||||
return random.randint(1, 65535)
|
||||
|
||||
def gen_ip(self, field):
|
||||
"""
|
||||
Generates an IP address.
|
||||
"""
|
||||
return RandIP()._fix()
|
||||
|
||||
def get_flags(self, field):
|
||||
"""
|
||||
Retrieves flags as a string.
|
||||
"""
|
||||
return str(self.layer.flags)
|
||||
|
||||
def set_flags(self, packet, field, value):
|
||||
"""
|
||||
Sets the flags field. There is a bug in scapy, if you retrieve an empty
|
||||
flags field, it will return "", but you cannot set this value back.
|
||||
To reproduce this bug:
|
||||
>>> setattr(IP(), "flags", str(IP().flags)) # raises a ValueError
|
||||
To handle this case, this method converts empty string to zero so that
|
||||
it can be safely stored.
|
||||
"""
|
||||
if value == "":
|
||||
value = 0
|
||||
self.layer.flags = value
|
||||
|
||||
def gen_flags(self, field):
|
||||
"""
|
||||
Generates random valid flags.
|
||||
"""
|
||||
sample = fuzz(self.protocol())
|
||||
|
||||
# Since scapy lazily evaluates fuzzing, we first must set a
|
||||
# legitimate value for scapy to evaluate what combination of flags it is
|
||||
sample.flags = sample.flags
|
||||
|
||||
return str(sample.flags)
|
||||
|
||||
|
||||
class TCPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access TCP header fields.
|
||||
"""
|
||||
name = "TCP"
|
||||
protocol = TCP
|
||||
_fields = [
|
||||
'sport',
|
||||
'dport',
|
||||
'seq',
|
||||
'ack',
|
||||
'dataofs',
|
||||
'reserved',
|
||||
'flags',
|
||||
'window',
|
||||
'chksum',
|
||||
'urgptr',
|
||||
'load',
|
||||
'options-eol',
|
||||
'options-nop',
|
||||
'options-mss',
|
||||
'options-wscale',
|
||||
'options-sackok',
|
||||
'options-sack',
|
||||
'options-timestamp',
|
||||
'options-altchksum',
|
||||
'options-altchksumopt',
|
||||
'options-md5header',
|
||||
'options-uto'
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
options_names = {
|
||||
"eol": 0,
|
||||
"nop": 1,
|
||||
"mss": 2,
|
||||
"wscale": 3,
|
||||
"sackok": 4,
|
||||
"sack": 5,
|
||||
#"echo" : 6,
|
||||
#"echo_reply" : 7,
|
||||
"timestamp": 8,
|
||||
"altchksum": 14,
|
||||
"altchksumopt": 15,
|
||||
"md5header": 19,
|
||||
#"quick_start" : 27,
|
||||
"uto": 28
|
||||
#"authentication": 29,
|
||||
#"experiment": 254
|
||||
}
|
||||
|
||||
# Each entry is Kind: length
|
||||
options_length = {
|
||||
0: 0, # EOL
|
||||
1: 0, # NOP
|
||||
2: 2, # MSS
|
||||
3: 1, # WScale
|
||||
4: 0, # SAckOK
|
||||
5: 0, # SAck
|
||||
6: 4, # Echo
|
||||
7: 4, # Echo Reply
|
||||
8: 8, # Timestamp
|
||||
14: 3, # AltChkSum
|
||||
15: 0, # AltChkSumOpt
|
||||
19: 16, # MD5header Option
|
||||
27: 6, # Quick-Start response
|
||||
28: 2, # User Timeout Option
|
||||
29: 4, # TCP Authentication Option
|
||||
254: 8, # Experiment
|
||||
|
||||
}
|
||||
# Required by scapy
|
||||
scapy_options = {
|
||||
0: "EOL",
|
||||
1: "NOP",
|
||||
2: "MSS",
|
||||
3: "WScale",
|
||||
4: "SAckOK",
|
||||
5: "SAck",
|
||||
8: "Timestamp",
|
||||
14: "AltChkSum",
|
||||
15: "AltChkSumOpt",
|
||||
28: "UTO",
|
||||
# 254:"Experiment" # scapy has two versions of this, so it doesn't work
|
||||
}
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the TCP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
# Special methods to help access fields that cannot be accessed normally
|
||||
self.getters = {
|
||||
'load' : self.get_load,
|
||||
'options' : self.get_options
|
||||
}
|
||||
self.setters = {
|
||||
'load' : self.set_load,
|
||||
'options' : self.set_options
|
||||
}
|
||||
# Special methods to help generate fields that cannot be generated normally
|
||||
self.generators = {
|
||||
'load' : self.gen_load,
|
||||
'dataofs' : self.gen_dataofs,
|
||||
'flags' : self.gen_flags,
|
||||
'chksum' : self.gen_chksum,
|
||||
'options' : self.gen_options
|
||||
}
|
||||
|
||||
|
||||
def gen_chksum(self, field):
|
||||
"""
|
||||
Generates a checksum.
|
||||
"""
|
||||
return random.randint(1, 65535)
|
||||
|
||||
def gen_dataofs(self, field):
|
||||
"""
|
||||
Generates a valid value for the data offset field.
|
||||
"""
|
||||
# Dataofs is a 4 bit header, so a max of 15
|
||||
return random.randint(1, 15)
|
||||
|
||||
def gen_flags(self, field):
|
||||
"""
|
||||
Generates a random set of flags. 50% of the time it picks randomly from
|
||||
a list of real flags, otherwise it returns fuzzed flags.
|
||||
"""
|
||||
if random.random() < 0.5:
|
||||
return random.choice(['S', 'A', 'SA', 'PA', 'FA', 'R', 'P', 'F', 'RA', ''])
|
||||
else:
|
||||
sample = fuzz(self.protocol())
|
||||
# Since scapy lazily evaluates fuzzing, we first must set a
|
||||
# legitimate value for scapy to evaluate what combination of flags it is
|
||||
sample.flags = sample.flags
|
||||
return str(sample.flags)
|
||||
|
||||
def get_options(self, field):
|
||||
"""
|
||||
Helper method to retrieve options.
|
||||
"""
|
||||
base, req_option = field.split("-")
|
||||
assert base == "options", "get_options can only be used to fetch options."
|
||||
option_type = self.option_str_to_int(req_option)
|
||||
i = 0
|
||||
# First, check if the option is already present in the packet
|
||||
for option in self.layer.options:
|
||||
# Scapy may try to be helpful and return the string of the option
|
||||
next_option = self.option_str_to_int(option[0])
|
||||
if option_type == next_option:
|
||||
_name, value = self.layer.options[i]
|
||||
# Some options (timestamp, checksums, nop) store their value in a
|
||||
# tuple.
|
||||
if isinstance(value, tuple):
|
||||
# Scapy returns values in any of these types
|
||||
if value in [None, b'', ()]:
|
||||
return ''
|
||||
value = value[0]
|
||||
if value in [None, b'', ()]:
|
||||
return ''
|
||||
if req_option == "md5header":
|
||||
return binascii.hexlify(value).decode("utf-8")
|
||||
|
||||
return value
|
||||
i += 1
|
||||
return ''
|
||||
|
||||
def set_options(self, packet, field, value):
|
||||
"""
|
||||
Helper method to set options.
|
||||
"""
|
||||
base, option = field.split("-")
|
||||
assert base == "options", "Must use an options field with set_options"
|
||||
|
||||
option_type = self.option_str_to_int(option)
|
||||
if type(value) == str:
|
||||
# Prepare the value for storage in the packet
|
||||
value = binascii.unhexlify(value)
|
||||
|
||||
# Scapy requires these options to be a tuple - since evaling this
|
||||
# is not yet supported, for now, SAck will always be an empty tuple
|
||||
if option in ["sack"]:
|
||||
value = ()
|
||||
# These options must be set as integers - if they didn't exist, they can
|
||||
# be added like this
|
||||
if option in ["timestamp", "mss", "wscale", "altchksum", "uto"] and not value:
|
||||
value = 0
|
||||
i = 0
|
||||
# First, check if the option is already present in the packet
|
||||
for option in self.layer.options:
|
||||
# Scapy may try to be helpful and return the string of the option
|
||||
next_option = self.option_str_to_int(option[0])
|
||||
|
||||
if option_type == next_option:
|
||||
packet["TCP"].options[i] = self.format_option(option_type, value)
|
||||
break
|
||||
i += 1
|
||||
# If we didn't break, the option doesn't exist in the packet currently.
|
||||
else:
|
||||
old_options_array = packet["TCP"].options
|
||||
old_options_array.append(self.format_option(option_type, value))
|
||||
packet["TCP"].options = old_options_array
|
||||
|
||||
# Let scapy recalculate the required values
|
||||
del self.layer.chksum
|
||||
del self.layer.dataofs
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].chksum
|
||||
del packet["IP"].len
|
||||
return True
|
||||
|
||||
def gen_options(self, field):
|
||||
"""
|
||||
Helper method to set options.
|
||||
"""
|
||||
_, option = field.split("-")
|
||||
option_num = self.options_names[option]
|
||||
length = self.options_length[option_num]
|
||||
|
||||
data = b''
|
||||
if length > 0:
|
||||
data = os.urandom(length)
|
||||
data = binascii.hexlify(data).decode()
|
||||
# MSS must be a 2-byte int
|
||||
if option_num == 2:
|
||||
data = random.randint(0, 65535)
|
||||
# WScale must be a 1-byte int
|
||||
elif option_num == 3:
|
||||
data = random.randint(0, 255)
|
||||
# Timestamp must be an int
|
||||
elif option_num == 8:
|
||||
data = random.randint(0, 4294967294)
|
||||
elif option_num == 14:
|
||||
data = random.randint(0, 255)
|
||||
elif option_num == 28:
|
||||
data = random.randint(0, 255)
|
||||
|
||||
return data
|
||||
|
||||
def option_str_to_int(self, option):
|
||||
"""
|
||||
Takes a string representation of an option and returns the option integer code.
|
||||
"""
|
||||
if type(option) == int:
|
||||
return option
|
||||
|
||||
assert "-" not in option, "Must be given specific option: %s." % option
|
||||
|
||||
for val in self.scapy_options:
|
||||
if self.scapy_options[val].lower() == option.lower():
|
||||
return val
|
||||
|
||||
if " " in option:
|
||||
option = option.replace(" ", "_").lower()
|
||||
|
||||
if option.lower() in self.options_names:
|
||||
return self.options_names[option.lower()]
|
||||
|
||||
def format_option(self, options_int, value):
|
||||
"""
|
||||
Formats the options so they will work with scapy.
|
||||
"""
|
||||
# NOPs
|
||||
if options_int == 1:
|
||||
return (self.scapy_options[options_int], ())
|
||||
elif options_int in [5]:
|
||||
return (self.scapy_options[options_int], value)
|
||||
# Timestamp
|
||||
elif options_int in [8, 14]:
|
||||
return (self.scapy_options[options_int], (value, 0))
|
||||
elif options_int in self.scapy_options:
|
||||
return (self.scapy_options[options_int], value)
|
||||
else:
|
||||
return (options_int, value)
|
||||
|
||||
|
||||
class UDPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access UDP header fields.
|
||||
"""
|
||||
name = "UDP"
|
||||
protocol = UDP
|
||||
_fields = [
|
||||
"sport",
|
||||
"dport",
|
||||
"chksum",
|
||||
"len",
|
||||
"load"
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the UDP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
'load' : self.get_load,
|
||||
}
|
||||
self.setters = {
|
||||
'load' : self.set_load,
|
||||
}
|
||||
self.generators = {
|
||||
'load' : self.gen_load,
|
||||
}
|
||||
|
||||
|
||||
class DNSLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access DNS header fields.
|
||||
"""
|
||||
name = "DNS"
|
||||
protocol = DNS
|
||||
_fields = [
|
||||
"id",
|
||||
"qr",
|
||||
"opcode",
|
||||
"aa",
|
||||
"tc",
|
||||
"rd",
|
||||
"ra",
|
||||
"z",
|
||||
"ad",
|
||||
"cd",
|
||||
"qd",
|
||||
"rcode",
|
||||
"qdcount",
|
||||
"ancount",
|
||||
"nscount",
|
||||
"arcount"
|
||||
]
|
||||
fields = _fields
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the DNS layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
|
||||
self.getters = {
|
||||
"qr" : self.get_bitfield,
|
||||
"aa" : self.get_bitfield,
|
||||
"tc" : self.get_bitfield,
|
||||
"rd" : self.get_bitfield,
|
||||
"ra" : self.get_bitfield,
|
||||
"z" : self.get_bitfield,
|
||||
"ad" : self.get_bitfield,
|
||||
"cd" : self.get_bitfield
|
||||
}
|
||||
|
||||
self.setters = {
|
||||
"qr" : self.set_bitfield,
|
||||
"aa" : self.set_bitfield,
|
||||
"tc" : self.set_bitfield,
|
||||
"rd" : self.set_bitfield,
|
||||
"ra" : self.set_bitfield,
|
||||
"z" : self.set_bitfield,
|
||||
"ad" : self.set_bitfield,
|
||||
"cd" : self.set_bitfield
|
||||
}
|
||||
|
||||
self.generators = {
|
||||
"id" : self.gen_id,
|
||||
"qr" : self.gen_bitfield,
|
||||
"opcode" : self.gen_opcode,
|
||||
"aa" : self.gen_bitfield,
|
||||
"tc" : self.gen_bitfield,
|
||||
"rd" : self.gen_bitfield,
|
||||
"ra" : self.gen_bitfield,
|
||||
"z" : self.gen_bitfield,
|
||||
"ad" : self.gen_bitfield,
|
||||
"cd" : self.gen_bitfield,
|
||||
"rcode" : self.gen_rcode,
|
||||
"qdcount" : self.gen_count,
|
||||
"ancount" : self.gen_count,
|
||||
"nscount" : self.gen_count,
|
||||
"arcount" : self.gen_count
|
||||
}
|
||||
|
||||
def get_bitfield(self, field):
|
||||
""""""
|
||||
return int(getattr(self.layer, field))
|
||||
|
||||
def set_bitfield(self, packet, field, value):
|
||||
""""""
|
||||
return setattr(self.layer, field, int(value))
|
||||
|
||||
def gen_bitfield(self, field):
|
||||
""""""
|
||||
return random.choice([0,1])
|
||||
|
||||
def gen_id(self, field):
|
||||
return random.randint(0, 65535)
|
||||
|
||||
def gen_opcode(self, field):
|
||||
return random.randint(0, 15)
|
||||
|
||||
def gen_rcode(self, field):
|
||||
return random.randint(0, 15)
|
||||
|
||||
def gen_count(self, field):
|
||||
return random.randint(0, 65535)
|
||||
|
||||
@staticmethod
|
||||
def dns_decompress(packet, logger):
|
||||
"""
|
||||
Performs DNS decompression on the given scapy packet, if applicable.
|
||||
Note that DNS compression/decompression must be done on the boundaries
|
||||
of a label, so DNS compression does not support arbitrary offsets.
|
||||
"""
|
||||
# If this is a TCP packet
|
||||
if packet.haslayer("TCP"):
|
||||
raise NotImplementedError
|
||||
|
||||
# Perform no action if this is not a DNS or DNSRQ packet
|
||||
if not packet.haslayer("DNS") or not packet.haslayer("DNSQR"):
|
||||
return packet
|
||||
|
||||
# Extract the query from the DNSQR layer
|
||||
query = packet["DNSQR"].qname.decode()
|
||||
if query[len(query) - 1] != '.':
|
||||
query += '.'
|
||||
|
||||
# Split the query by label
|
||||
labels = query.split(".")
|
||||
|
||||
# Collect the first and second half of the query
|
||||
fhalf = labels[0]
|
||||
shalf = ".".join(labels[1:])
|
||||
|
||||
# Build the first DNS query directly. The format of this a byte string like this:
|
||||
# b'\x07minghui\xc0\x1a\x00\x01\x00\x01'
|
||||
# \x07 = the length of the label in this DNSQR
|
||||
# minghui = the portion of the domain we will request in the first DNSQR
|
||||
# \xc0\x1a = offset into the DNS packet where the rest of the query will be. The actual offset
|
||||
# here is the \x1a - DNS mandates that if compression is used, the first two bits be 11
|
||||
# to differentiate them from the rest. \x1A = 26, which is the length of the DNS header
|
||||
# plus the length of this DNSQR.
|
||||
# \x00\x01 = type A record
|
||||
# \x00\x01 = IN
|
||||
length = bytes([len(fhalf)])
|
||||
label = fhalf.encode()
|
||||
|
||||
# Since the domain will include an extra ".", add 1
|
||||
# 2 * 6 is the DNS header
|
||||
# 1 is the byte that determines the length of the label
|
||||
# len(label) is the length of the label
|
||||
# 2 is the offset pointer
|
||||
# 4 - other record information (class, IN)
|
||||
packet_offset = 2 * 6 + 1 + len(label) + 2 + 2 + 2
|
||||
|
||||
# The word must start with binary 11, so OR the offset with 0xC000.
|
||||
offset = (0xc000 | packet_offset).to_bytes(2, byteorder='big')
|
||||
request = b'\x00\x01\x00\x01'
|
||||
|
||||
dns_qr1 = length + label + offset + request
|
||||
|
||||
# Build the second DNS query directly. The format of the byte string is the same as above
|
||||
# b'\x02ca\x00\x00\x01\x00\x01'
|
||||
# \x02 = length of the remaining domain
|
||||
# ca = portion of the domain in this DNSQR
|
||||
# \x00 = null byte to signify the end of the query
|
||||
# \x00\x01 = type A record
|
||||
# \x00\x01 = IN
|
||||
# Since the second half could potentially contain many labels, this is done in a list comprehension
|
||||
dns_qr2 = b"".join([bytes([len(tld)]) + tld.encode() for tld in shalf.split(".")]) + b"\x00\x01\x00\x01"
|
||||
|
||||
# Next, we must rebuild the DNS packet itself. If we try to have scapy parse either dns_qr1 or dns_qr2, they
|
||||
# will look malformed, since neither contains a complete request. Therefore, we must build the entire
|
||||
# DNS packet at once. First, we must remove the original DNSQR, since this contains the original request
|
||||
del packet["DNS"].qd
|
||||
|
||||
# Once the DNSQR is removed, scapy automatically sets the qdcount to 0. Adjust it to 2
|
||||
packet["DNS"].qdcount = 2
|
||||
|
||||
# Extract the DNS header standalone now for building
|
||||
dns_header = bytes(packet["DNS"])
|
||||
|
||||
dns_packet = DNS(dns_header + dns_qr1 + dns_qr2)
|
||||
|
||||
del packet["DNS"]
|
||||
packet = packet / dns_packet
|
||||
|
||||
# Since the size and data of the packet have changed, force scapy to recalculate the important fields
|
||||
# in below layers, if applicable
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].chksum
|
||||
del packet["IP"].len
|
||||
if packet.haslayer("UDP"):
|
||||
del packet["UDP"].chksum
|
||||
del packet["UDP"].len
|
||||
|
||||
return packet
|
||||
|
||||
|
||||
class DNSQRLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access DNSQR header fields.
|
||||
"""
|
||||
name = "DNSQR"
|
||||
protocol = DNSQR
|
||||
_fields = [
|
||||
"qname",
|
||||
"qtype",
|
||||
"qclass"
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the DNS layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
"qname" : self.get_qname
|
||||
}
|
||||
self.generators = {
|
||||
"qname" : self.gen_qname
|
||||
}
|
||||
|
||||
def get_qname(self, field):
|
||||
"""
|
||||
Returns decoded qname from packet.
|
||||
"""
|
||||
return self.layer.qname.decode('utf-8')
|
||||
|
||||
def gen_qname(self, field):
|
||||
"""
|
||||
Generates domain name.
|
||||
"""
|
||||
return "example.com."
|
||||
|
||||
@classmethod
|
||||
def name_matches(cls, name):
|
||||
"""
|
||||
Scapy returns the name of DNSQR as _both_ DNSQR and "DNS Question Record",
|
||||
which breaks parsing. Override the name_matches method to handle that case
|
||||
here.
|
||||
"""
|
||||
return name.upper() in ["DNSQR", "DNS QUESTION RECORD"]
|
|
@ -1,8 +1,21 @@
|
|||
from actions.action import Action
|
||||
|
||||
|
||||
class SleepAction(Action):
|
||||
"""
|
||||
Defines the SleepAction - causes the engine to pause before sending a packet.
|
||||
"""
|
||||
# Do not select the sleep action during evolutions
|
||||
frequency = 0
|
||||
def __init__(self, time=1, environment_id=None):
|
||||
Action.__init__(self, "sleep", "both")
|
||||
"""
|
||||
Initializes the sleep action.
|
||||
|
||||
Args:
|
||||
time (float): How much time the packet should delay before sending
|
||||
environment_id (str, optional): Environment ID of the strategy this action is a part of
|
||||
"""
|
||||
Action.__init__(self, "sleep", "out")
|
||||
self.terminal = False
|
||||
self.branching = False
|
||||
self.time = time
|
||||
|
|
|
@ -0,0 +1,85 @@
|
|||
import threading
|
||||
import os
|
||||
|
||||
import layers.packet
|
||||
from scapy.all import sniff
|
||||
from scapy.utils import PcapWriter
|
||||
|
||||
|
||||
class Sniffer():
|
||||
"""
|
||||
The sniffer class lets the user begin and end sniffing whenever in a given location with a port to filter on.
|
||||
Call start_sniffing to begin sniffing and stop_sniffing to stop sniffing.
|
||||
"""
|
||||
|
||||
def __init__(self, location, port, logger):
|
||||
"""
|
||||
Intializes a sniffer object.
|
||||
Needs a location and a port to filter on.
|
||||
"""
|
||||
self.stop_sniffing_flag = False
|
||||
self.location = location
|
||||
self.port = port
|
||||
self.pcap_thread = None
|
||||
self.packet_dumper = None
|
||||
self.logger = logger
|
||||
full_path = os.path.dirname(location)
|
||||
assert port, "Need to specify a port in order to launch a sniffer"
|
||||
if not os.path.exists(full_path):
|
||||
os.makedirs(full_path)
|
||||
|
||||
def __packet_callback(self, scapy_packet):
|
||||
"""
|
||||
This callback is called whenever a packet is applied.
|
||||
Returns true if it should finish, otherwise, returns false.
|
||||
"""
|
||||
packet = layers.packet.Packet(scapy_packet)
|
||||
for proto in ["TCP", "UDP"]:
|
||||
if(packet.haslayer(proto) and ((packet[proto].sport == self.port) or (packet[proto].dport == self.port))):
|
||||
break
|
||||
else:
|
||||
return self.stop_sniffing_flag
|
||||
|
||||
self.logger.debug(str(packet))
|
||||
self.packet_dumper.write(scapy_packet)
|
||||
return self.stop_sniffing_flag
|
||||
|
||||
def __spawn_sniffer(self):
|
||||
"""
|
||||
Saves pcaps to a file. Should be run as a thread.
|
||||
Ends when the stop_sniffing_flag is set. Should not be called by user
|
||||
"""
|
||||
self.packet_dumper = PcapWriter(self.location, append=True, sync=True)
|
||||
while(self.stop_sniffing_flag == False):
|
||||
sniff(stop_filter=self.__packet_callback, timeout=1)
|
||||
|
||||
def start_sniffing(self):
|
||||
"""
|
||||
Starts sniffing. Should be called by user.
|
||||
"""
|
||||
self.stop_sniffing_flag = False
|
||||
self.pcap_thread = threading.Thread(target=self.__spawn_sniffer)
|
||||
self.pcap_thread.start()
|
||||
self.logger.debug("Sniffer starting to port %d" % self.port)
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
Defines a context manager for this sniffer; simply starts sniffing.
|
||||
"""
|
||||
self.start_sniffing()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, tb):
|
||||
"""
|
||||
Defines exit context manager behavior for this sniffer; simply stops sniffing.
|
||||
"""
|
||||
self.stop_sniffing()
|
||||
|
||||
def stop_sniffing(self):
|
||||
"""
|
||||
Stops the sniffer by setting the flag and calling join
|
||||
"""
|
||||
if(self.pcap_thread):
|
||||
self.stop_sniffing_flag = True
|
||||
self.pcap_thread.join()
|
||||
self.logger.debug("Sniffer stopping")
|
|
@ -8,6 +8,8 @@ class Strategy(object):
|
|||
def __init__(self, in_actions, out_actions, environment_id=None):
|
||||
self.in_actions = in_actions
|
||||
self.out_actions = out_actions
|
||||
self.descendents = []
|
||||
|
||||
self.in_enabled = True
|
||||
self.out_enabled = True
|
||||
|
||||
|
@ -52,6 +54,48 @@ class Strategy(object):
|
|||
rep += "%s\n" % action_tree.pretty_print()
|
||||
return rep
|
||||
|
||||
def initialize(self, logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, disabled=None):
|
||||
"""
|
||||
Initializes a new strategy object randomly.
|
||||
"""
|
||||
# Disable specific forests if none requested
|
||||
if num_in_trees == 0:
|
||||
self.in_enabled = False
|
||||
|
||||
if num_out_trees == 0:
|
||||
self.out_enabled = False
|
||||
|
||||
# If a specific population seed is requested, build using that
|
||||
if seed:
|
||||
starting_strat = actions.utils.parse(seed, logger)
|
||||
self.out_actions = starting_strat.out_actions
|
||||
self.in_actions = starting_strat.in_actions
|
||||
return self
|
||||
|
||||
self.init_from_scratch(num_in_trees, num_out_trees, num_in_actions, num_out_actions, disabled=disabled)
|
||||
return self
|
||||
|
||||
def init_from_scratch(self, num_in_trees, num_out_trees, num_in_actions, num_out_actions, disabled=None):
|
||||
"""
|
||||
Initializes this individual by drawing random actions.
|
||||
"""
|
||||
for _ in range(0, num_in_trees):
|
||||
# Define a new in action tree
|
||||
in_tree = actions.tree.ActionTree("in")
|
||||
# Initialize the in tree
|
||||
in_tree.initialize(num_in_actions, self.environment_id, disabled=disabled)
|
||||
# Add them to this strategy
|
||||
self.in_actions.append(in_tree)
|
||||
|
||||
for _ in range(0, num_out_trees):
|
||||
# Define a new out action tree
|
||||
out_tree = actions.tree.ActionTree("out")
|
||||
# Initialize the out tree
|
||||
out_tree.initialize(num_out_actions, self.environment_id, disabled=disabled)
|
||||
# Add them to this strategy
|
||||
self.out_actions.append(out_tree)
|
||||
|
||||
|
||||
def act_on_packet(self, packet, logger, direction="out"):
|
||||
"""
|
||||
Runs the strategy on a given scapy packet.
|
||||
|
@ -62,6 +106,7 @@ class Strategy(object):
|
|||
return [packet]
|
||||
return self.run_on_packet(packet, logger, direction)
|
||||
|
||||
|
||||
def run_on_packet(self, packet, logger, direction):
|
||||
"""
|
||||
Runs the strategy on a given packet given the forest direction.
|
||||
|
@ -86,3 +131,99 @@ class Strategy(object):
|
|||
if not ran:
|
||||
packets_to_send = [packet]
|
||||
return packets_to_send
|
||||
|
||||
def mutate_dir(self, trees, direction, logger):
|
||||
"""
|
||||
Mutates a list of trees. Requires the direction the tree operates on
|
||||
(in or out).
|
||||
"""
|
||||
pick = random.uniform(0, 1)
|
||||
if pick < 0.1 or not trees:
|
||||
new_tree = actions.tree.ActionTree(direction)
|
||||
new_tree.initialize(1, self.environment_id)
|
||||
trees.append(new_tree)
|
||||
elif pick < 0.2 and trees:
|
||||
trees.remove(random.choice(trees))
|
||||
elif pick < 0.25 and trees and len(trees) > 1:
|
||||
random.shuffle(trees)
|
||||
else:
|
||||
for action_tree in trees:
|
||||
action_tree.mutate()
|
||||
|
||||
def mutate(self, logger):
|
||||
"""
|
||||
Top level mutation function for a strategy. Simply mutates the out
|
||||
and in trees.
|
||||
"""
|
||||
if self.in_enabled:
|
||||
self.mutate_dir(self.in_actions, "in", logger)
|
||||
if self.out_enabled:
|
||||
self.mutate_dir(self.out_actions, "out", logger)
|
||||
return self
|
||||
|
||||
|
||||
def swap_one(forest1, forest2):
|
||||
"""
|
||||
Swaps a random tree from forest1 and forest2.
|
||||
|
||||
It picks a random element within forest1 and a random element within forest2,
|
||||
chooses a random index within each forest, and inserts the random element
|
||||
"""
|
||||
assert type(forest1) == list
|
||||
assert type(forest2) == list
|
||||
rand_idx1, rand_idx2 = 0, 0
|
||||
donation, other_donation = None, None
|
||||
if forest1:
|
||||
donation = random.choice(forest1)
|
||||
forest1.remove(donation)
|
||||
if len(forest1) > 0:
|
||||
rand_idx1 = random.choice(list(range(0, len(forest1))))
|
||||
|
||||
if forest2:
|
||||
other_donation = random.choice(forest2)
|
||||
forest2.remove(other_donation)
|
||||
if len(forest2) > 0:
|
||||
rand_idx2 = random.choice(list(range(0, len(forest2))))
|
||||
|
||||
if other_donation:
|
||||
forest1.insert(rand_idx1, other_donation)
|
||||
|
||||
if donation:
|
||||
forest2.insert(rand_idx2, donation)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def do_mate(forest1, forest2):
|
||||
"""
|
||||
Performs mating between two given forests (lists of trees).
|
||||
With 80% probability, a random tree from each forest are mated,
|
||||
otherwise, a random tree is swapped between them.
|
||||
"""
|
||||
# If 80% and there are trees in both forests to mate, or
|
||||
# if there is only 1 tree in each forest, mate those trees
|
||||
if (random.random() < 0.8 and forest1 and forest2) or \
|
||||
(len(forest1) == 1 and len(forest2) == 1):
|
||||
tree1 = random.choice(forest1)
|
||||
tree2 = random.choice(forest2)
|
||||
return tree1.mate(tree2)
|
||||
# Otherwise, swap a random tree from each forest
|
||||
elif forest1 or forest2:
|
||||
return swap_one(forest1, forest2)
|
||||
return False
|
||||
|
||||
|
||||
def mate(ind1, ind2, indpb):
|
||||
"""
|
||||
Executes a uniform crossover that modify in place the two
|
||||
individuals. The attributes are swapped according to the
|
||||
*indpb* probability.
|
||||
"""
|
||||
out_success, in_success = True, True
|
||||
if ind1.out_enabled and random.random() < indpb:
|
||||
out_success = do_mate(ind1.out_actions, ind2.out_actions)
|
||||
|
||||
if ind1.in_enabled and random.random() < indpb:
|
||||
in_success = do_mate(ind1.in_actions, ind2.in_actions)
|
||||
|
||||
return out_success and in_success
|
||||
|
|
|
@ -3,16 +3,16 @@ TamperAction
|
|||
|
||||
One of the four packet-level primitives supported by Geneva. Responsible for any packet-level
|
||||
modifications (particularly header modifications). It supports the following primitives:
|
||||
- no operation: it returns the packet given
|
||||
- replace: it changes a packet field to a fixed value
|
||||
- corrupt: it changes a packet field to a randomly generated value each time it is run
|
||||
- add: adds a given value to the value in a field
|
||||
- compress: performs DNS decompression on the packet (if applicable)
|
||||
- no operation: it returns the packet given
|
||||
- replace: it changes a packet field to a fixed value
|
||||
- corrupt: it changes a packet field to a randomly generated value each time it is run
|
||||
- add: adds a given value to the value in a field
|
||||
- compress: performs DNS decompression on the packet (if applicable)
|
||||
"""
|
||||
|
||||
from actions.action import Action
|
||||
import actions.utils
|
||||
from actions.layer import DNSLayer
|
||||
from layers.dns_layer import DNSLayer
|
||||
|
||||
import random
|
||||
|
||||
|
@ -20,18 +20,72 @@ import random
|
|||
# All supported tamper primitives
|
||||
SUPPORTED_PRIMITIVES = ["corrupt", "replace", "add", "compress"]
|
||||
|
||||
# Tamper primitives we can mutate to by default
|
||||
ACTIVATED_PRIMITIVES = ["replace", "corrupt", "add"]
|
||||
|
||||
|
||||
class TamperAction(Action):
|
||||
"""
|
||||
Defines the TamperAction for Geneva.
|
||||
"""
|
||||
frequency = 5
|
||||
def __init__(self, environment_id=None, field=None, tamper_type=None, tamper_value=None, tamper_proto="TCP"):
|
||||
"""
|
||||
Creates a tamper object.
|
||||
|
||||
Args:
|
||||
environment_id (str, optional): environment_id of a previously run strategy, used to find packet captures
|
||||
field (str, optional): field that the object will tamper. If not set, all the parameters are chosen randomly
|
||||
tamper_type (str, optional): primitive this tamper will use ("corrupt")
|
||||
tamper_value (str, optional): value to tamper to
|
||||
tamper_proto (str, optional): protocol we are tampering
|
||||
"""
|
||||
Action.__init__(self, "tamper", "both")
|
||||
self.field = field
|
||||
self.tamper_value = tamper_value
|
||||
self.tamper_proto = actions.utils.string_to_protocol(tamper_proto)
|
||||
self.tamper_proto_str = tamper_proto
|
||||
|
||||
self.tamper_type = tamper_type
|
||||
if not self.tamper_type:
|
||||
self._mutate_tamper_type()
|
||||
|
||||
if not self.field:
|
||||
self._mutate(environment_id)
|
||||
|
||||
def mutate(self, environment_id=None):
|
||||
"""
|
||||
Mutate can switch between the tamper type, field.
|
||||
"""
|
||||
# With some probability switch tamper types
|
||||
pick = random.random()
|
||||
if pick < 0.2:
|
||||
self._mutate_tamper_type()
|
||||
else:
|
||||
self._mutate(environment_id)
|
||||
|
||||
def _mutate_tamper_type(self):
|
||||
"""
|
||||
Randomly picks a tamper type to change to.
|
||||
"""
|
||||
self.tamper_type = random.choice(ACTIVATED_PRIMITIVES)
|
||||
if self.tamper_type == "compress":
|
||||
self.tamper_proto_str = "DNS"
|
||||
self.tamper_proto = actions.utils.string_to_protocol(self.tamper_proto_str)
|
||||
self.field = "qd"
|
||||
|
||||
def _mutate(self, environment_id):
|
||||
"""
|
||||
Mutates this action using:
|
||||
- previously seen packets with 50% probability
|
||||
- a fuzzed packet with 50% probability
|
||||
"""
|
||||
# Retrieve a new protocol and field options for this protocol
|
||||
proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, 0.5)
|
||||
self.tamper_proto = proto
|
||||
self.tamper_proto_str = proto.__name__
|
||||
self.field = field
|
||||
self.tamper_value = value
|
||||
|
||||
def tamper(self, packet, logger):
|
||||
"""
|
||||
|
|
|
@ -15,8 +15,19 @@ class TraceAction(Action):
|
|||
TraceAction is an experimental action that is never used
|
||||
in actual evolution
|
||||
"""
|
||||
# Do not select Trace during evolutions
|
||||
frequency = 0
|
||||
def __init__(self, start_ttl=1, end_ttl=64, environment_id=None):
|
||||
"""
|
||||
Initializes the trace action.
|
||||
|
||||
Args:
|
||||
start_ttl (int): Starting TTL to use
|
||||
end_ttl (int): TTL to end with
|
||||
environment_id (str, optional): Environment ID associated with the strategy we are a part of
|
||||
"""
|
||||
Action.__init__(self, "trace", "out")
|
||||
self.enabled = True
|
||||
self.terminal = True
|
||||
self.branching = False
|
||||
self.start_ttl = start_ttl
|
||||
|
@ -40,7 +51,7 @@ class TraceAction(Action):
|
|||
return packet, None
|
||||
|
||||
if self.ran:
|
||||
logger.debug(" - trace action already ran. Dropping given traffic. To reset this action, restart the engine.")
|
||||
logger.debug(" - trace action already ran. Dropping given traffic.")
|
||||
return None, None
|
||||
|
||||
self.ran = True
|
||||
|
|
|
@ -24,12 +24,35 @@ class ActionTree():
|
|||
"""
|
||||
|
||||
def __init__(self, direction, trigger=None):
|
||||
"""
|
||||
Creates this action tree.
|
||||
|
||||
Args:
|
||||
direction (str): Direction this tree is facing ("out", "in")
|
||||
trigger (:obj:`actions.trigger.Trigger`): Trigger to use with this tree
|
||||
"""
|
||||
self.trigger = trigger
|
||||
self.action_root = None
|
||||
self.direction = direction
|
||||
self.environment_id = None
|
||||
self.ran = False
|
||||
|
||||
def initialize(self, num_actions, environment_id, allow_terminal=True, disabled=None):
|
||||
"""
|
||||
Sets up this action tree with a given number of random actions.
|
||||
Note that the returned action trees may have less actions than num_actions
|
||||
if terminal actions are used.
|
||||
"""
|
||||
self.environment_id = environment_id
|
||||
self.trigger = actions.trigger.Trigger(None, None, None, environment_id=environment_id)
|
||||
if not allow_terminal or random.random() > 0.1:
|
||||
allow_terminal = False
|
||||
|
||||
for _ in range(num_actions):
|
||||
new_action = self.get_rand_action(self.direction, disabled=disabled)
|
||||
self.add_action(new_action)
|
||||
return self
|
||||
|
||||
def __iter__(self):
|
||||
"""
|
||||
Sets up a preoder iterator for the tree.
|
||||
|
@ -222,8 +245,8 @@ class ActionTree():
|
|||
if not node.right:
|
||||
yield right_packet
|
||||
|
||||
# If we have a left action and were given a packet to pass on, run
|
||||
# on the left packet
|
||||
# If we have a right action and were given a packet to pass on, run
|
||||
# on the right packet
|
||||
if node.right and right_packet:
|
||||
for rpacket in self.do_run(node.right, right_packet, logger):
|
||||
yield rpacket
|
||||
|
@ -383,6 +406,22 @@ class ActionTree():
|
|||
break
|
||||
return action_added
|
||||
|
||||
def get_rand_action(self, direction, request=None, allow_terminal=True, disabled=None):
|
||||
"""
|
||||
Retrieves and initializes a random action that can run in the given direction.
|
||||
"""
|
||||
pick = random.random()
|
||||
action_options = actions.action.Action.get_actions(direction, disabled=disabled, allow_terminal=allow_terminal)
|
||||
# Check to make sure there are still actions available to use
|
||||
assert action_options, "No actions were available"
|
||||
act_dict = {}
|
||||
all_opts = []
|
||||
for action_name, act_cls in action_options:
|
||||
act_dict[action_name] = act_cls
|
||||
all_opts += ([act_cls] * act_cls.frequency)
|
||||
new_action = act_dict.get(request, random.choice(all_opts))
|
||||
return new_action(environment_id=self.environment_id)
|
||||
|
||||
def remove_one(self):
|
||||
"""
|
||||
Removes a random leaf from the tree.
|
||||
|
@ -392,6 +431,26 @@ class ActionTree():
|
|||
action = random.choice(self)
|
||||
return self.remove_action(action)
|
||||
|
||||
def mutate(self):
|
||||
"""
|
||||
Mutates this action tree with respect to a given direction.
|
||||
"""
|
||||
pick = random.uniform(0, 1)
|
||||
if pick < 0.20 or not self.action_root:
|
||||
new_action = self.get_rand_action(direction=self.direction)
|
||||
self.add_action(new_action)
|
||||
elif pick < 0.65 and self.action_root:
|
||||
action = random.choice(self)
|
||||
action.mutate(environment_id=self.environment_id)
|
||||
# If this individual has never been run under the evaluator,
|
||||
# or if it ran and it failed, it won't have an environment_id,
|
||||
# which means it has no saved packets to read from.
|
||||
elif pick < 0.80 and self.environment_id:
|
||||
self.trigger.mutate(self.environment_id)
|
||||
else:
|
||||
self.remove_one()
|
||||
return self
|
||||
|
||||
def choose_one(self):
|
||||
"""
|
||||
Picks a random element in the tree.
|
||||
|
@ -415,6 +474,40 @@ class ActionTree():
|
|||
return action, "right"
|
||||
return None, None
|
||||
|
||||
def swap(self, my_donation, other_tree, other_donation):
|
||||
"""
|
||||
Swaps a node in this tree with a node in another tree.
|
||||
"""
|
||||
parent, direction = self.get_parent(my_donation)
|
||||
other_parent, other_direction = other_tree.get_parent(other_donation)
|
||||
# If this tree is empty or I'm trying to donate my root
|
||||
if not my_donation or not parent:
|
||||
parent = self
|
||||
direction = "action_root"
|
||||
# if the other tree is empty or they are trying to donate their root
|
||||
if not other_donation or not other_parent:
|
||||
other_parent = other_tree
|
||||
other_direction = "action_root"
|
||||
|
||||
setattr(parent, direction, other_donation)
|
||||
setattr(other_parent, other_direction, my_donation)
|
||||
|
||||
return True
|
||||
|
||||
def mate(self, other_tree):
|
||||
"""
|
||||
Mates this tree with another tree.
|
||||
"""
|
||||
# If both trees are empty, nothing to do
|
||||
if not self.action_root and not other_tree.action_root:
|
||||
return False
|
||||
|
||||
# Chose an action node in this tree to swap
|
||||
my_swap_node = self.choose_one()
|
||||
other_swap_node = other_tree.choose_one()
|
||||
|
||||
return self.swap(my_swap_node, other_tree, other_swap_node)
|
||||
|
||||
def pretty_print_help(self, root, visual=False, parent=None):
|
||||
"""
|
||||
Pretty prints the tree.
|
||||
|
@ -448,6 +541,7 @@ class ActionTree():
|
|||
|
||||
return newroot
|
||||
|
||||
|
||||
def pretty_print(self, visual=False):
|
||||
"""
|
||||
Pretty prints the tree.
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import actions.utils
|
||||
import random
|
||||
import re
|
||||
import layers.packet
|
||||
|
||||
FIXED_TRIGGER = None
|
||||
GAS_ENABLED = True
|
||||
|
@ -15,6 +16,7 @@ class Trigger(object):
|
|||
- trigger_value: the value in the trigger_field that, upon a match, will cause the trigger to fire
|
||||
- environment_id: environment_id the current trigger is running under. Used to retrieve previously saved packets
|
||||
- gas: how many times this trigger can fire before it stops triggering. gas=None disables gas (unlimited triggers.)
|
||||
- has_wildcard: represents if the trigger will match a specific value, or any value containing trigger_value
|
||||
"""
|
||||
self.trigger_type = trigger_type
|
||||
self.trigger_field = trigger_field
|
||||
|
@ -23,9 +25,55 @@ class Trigger(object):
|
|||
self.environment_id = environment_id
|
||||
self.num_seen = 0
|
||||
self.gas_remaining = gas
|
||||
self.has_wildcard = False
|
||||
# Bomb triggers act like reverse triggers. They run the action only after the action has been triggered x times
|
||||
self.bomb_trigger = bool(gas and gas < 0)
|
||||
self.ran = False
|
||||
# ignore numerical trigger values
|
||||
if isinstance(self.trigger_value, (str)):
|
||||
# check if value field is wildcarded or not
|
||||
if(len(self.trigger_value) != 0 and self.trigger_value[-1] == '*'):
|
||||
self.has_wildcard = True
|
||||
# remove '*' wildcard from trigger_value for ease of use
|
||||
self.trigger_value = self.trigger_value[:-1]
|
||||
if not self.trigger_type:
|
||||
self.trigger_type, self.trigger_proto, self.trigger_field, self.trigger_value, self.gas_remaining = Trigger.get_rand_trigger(environment_id, 1)
|
||||
|
||||
@staticmethod
|
||||
def get_gas():
|
||||
"""
|
||||
Returns a random value for gas for this trigger.
|
||||
"""
|
||||
if GAS_ENABLED and random.random() < 0.2:
|
||||
# Use gas in 20% of scenarios
|
||||
# Pick a number for gas between 0 - 5
|
||||
gas_remaining = int(random.random() * 5)
|
||||
else:
|
||||
# Do not use gas
|
||||
gas_remaining = None
|
||||
return gas_remaining
|
||||
|
||||
@staticmethod
|
||||
def get_rand_trigger(environment_id, real_packet_probability):
|
||||
"""
|
||||
Creates a random trigger.
|
||||
"""
|
||||
proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, real_packet_probability, enable_options=False, enable_load=False)
|
||||
gas_remaining = Trigger.get_gas()
|
||||
if not FIXED_TRIGGER:
|
||||
# Only "field" triggers are supported currently
|
||||
return "field", proto.__name__, field, value, gas_remaining
|
||||
return (FIXED_TRIGGER.trigger_type,
|
||||
FIXED_TRIGGER.trigger_proto,
|
||||
FIXED_TRIGGER.trigger_field,
|
||||
FIXED_TRIGGER.trigger_value,
|
||||
FIXED_TRIGGER.gas_remaining)
|
||||
|
||||
def mutate(self, environment_id, real_packet_probability=0.5):
|
||||
"""
|
||||
Mutates this trigger object by picking a new protocol, field, and value.
|
||||
"""
|
||||
self.trigger_type, self.trigger_proto, self.trigger_field, self.trigger_value, self.gas_remaining = Trigger.get_rand_trigger(environment_id, real_packet_probability)
|
||||
|
||||
def is_applicable(self, packet, logger):
|
||||
"""
|
||||
|
@ -37,6 +85,9 @@ class Trigger(object):
|
|||
return False
|
||||
|
||||
packet_value = packet.get(self.trigger_proto, self.trigger_field)
|
||||
if self.has_wildcard:
|
||||
will_run = (self.trigger_value in packet_value)
|
||||
else:
|
||||
will_run = (self.trigger_value == packet_value)
|
||||
|
||||
# Track if this action is used
|
||||
|
@ -127,7 +178,7 @@ class Trigger(object):
|
|||
value = m.group(3)
|
||||
|
||||
# Parse out the given value if necessary
|
||||
value = actions.packet.Packet.parse(proto, field, value)
|
||||
value = layers.packet.Packet.parse(proto, field, value)
|
||||
|
||||
# Trigger gas is set to None if it is disabled
|
||||
trigger_gas = None
|
||||
|
|
351
actions/utils.py
351
actions/utils.py
|
@ -2,34 +2,53 @@ import copy
|
|||
import datetime
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import string
|
||||
import sys
|
||||
import random
|
||||
import urllib.parse
|
||||
import re
|
||||
import socket
|
||||
import random
|
||||
import struct
|
||||
|
||||
import actions.action
|
||||
import actions.trigger
|
||||
import actions.packet
|
||||
import layers.packet
|
||||
import plugins.plugin_client
|
||||
import plugins.plugin_server
|
||||
|
||||
from scapy.all import TCP, IP, UDP, rdpcap
|
||||
import netifaces
|
||||
|
||||
|
||||
RUN_DIRECTORY = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
RUN_DIRECTORY = os.path.join("trials", datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
|
||||
|
||||
# Hard coded options
|
||||
FLAGFOLDER = "flags"
|
||||
|
||||
# Holds copy of console file handler's log level
|
||||
CONSOLE_LOG_LEVEL = logging.DEBUG
|
||||
CONSOLE_LOG_LEVEL = "debug"
|
||||
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(BASEPATH)
|
||||
|
||||
|
||||
class SkipStrategyException(Exception):
|
||||
"""
|
||||
Raised to signal that this strategy evaluation should be cut off.
|
||||
"""
|
||||
def __init__(self, msg, fitness):
|
||||
"""
|
||||
Creates the exception with the fitness to pass back
|
||||
"""
|
||||
self.fitness = fitness
|
||||
self.msg = msg
|
||||
|
||||
|
||||
def parse(requested_trees, logger):
|
||||
"""
|
||||
Parses a string representation of a solution into its object form.
|
||||
|
@ -71,7 +90,9 @@ def parse(requested_trees, logger):
|
|||
# strategies, so restore the "|" that was lost from the split
|
||||
str_action = str_action + "|"
|
||||
new_tree = actions.tree.ActionTree(direction)
|
||||
new_tree.parse(str_action, logger)
|
||||
success = new_tree.parse(str_action, logger)
|
||||
if success is False:
|
||||
raise actions.tree.ActionTreeParseError("Failed to parse tree")
|
||||
|
||||
# Once all the actions are parsed, add this tree to the
|
||||
# current direction of actions
|
||||
|
@ -85,7 +106,8 @@ def parse(requested_trees, logger):
|
|||
return strat
|
||||
|
||||
|
||||
def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_level=logging.DEBUG):
|
||||
|
||||
def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_level="DEBUG", demo_mode=False):
|
||||
"""
|
||||
Configures and returns a logger.
|
||||
"""
|
||||
|
@ -100,7 +122,7 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev
|
|||
os.makedirs(flag_path)
|
||||
# Set up a client logger
|
||||
logger = logging.getLogger(logger_name + environment_id)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.setLevel("DEBUG")
|
||||
# Disable the root logger to avoid double printing
|
||||
logger.propagate = False
|
||||
|
||||
|
@ -108,7 +130,7 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev
|
|||
if logger.handlers:
|
||||
return logger
|
||||
fh = logging.FileHandler(os.path.join(basepath, log_dir, "logs", "%s.%s.log" % (environment_id, log_name)))
|
||||
fh.setLevel(logging.DEBUG)
|
||||
fh.setLevel("DEBUG")
|
||||
|
||||
log_prefix = "[%s] " % log_name.upper()
|
||||
formatter = logging.Formatter("%(asctime)s %(levelname)s:" + log_prefix + "%(message)s", datefmt="%Y-%m-%d %H:%M:%S")
|
||||
|
@ -119,10 +141,95 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev
|
|||
ch = logging.StreamHandler()
|
||||
ch.setFormatter(formatter)
|
||||
ch.setLevel(log_level)
|
||||
CONSOLE_LOG_LEVEL = ch.level
|
||||
CONSOLE_LOG_LEVEL = log_level.lower()
|
||||
logger.addHandler(ch)
|
||||
return logger
|
||||
return CustomAdapter(logger, {}) if demo_mode else logger
|
||||
|
||||
class CustomAdapter(logging.LoggerAdapter):
|
||||
"""
|
||||
Used for demo mode, to change sensitive IP addresses where necessary. Can be used (mostly) like a regular logger.
|
||||
"""
|
||||
regex = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
|
||||
|
||||
def __init__(self, logger, extras):
|
||||
super().__init__(logger, extras)
|
||||
self.handlers = logger.handlers
|
||||
self.ips = {}
|
||||
|
||||
def debug(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Print a debug message, uses logger.debug.
|
||||
"""
|
||||
msg, args, kwargs = self.process(msg, args, kwargs)
|
||||
|
||||
self.logger.debug(msg, *args, **kwargs)
|
||||
|
||||
def info(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Print an info message, uses logger.info.
|
||||
"""
|
||||
msg, args, kwargs = self.process(msg, args, kwargs)
|
||||
|
||||
self.logger.info(msg, *args, **kwargs)
|
||||
|
||||
def warning(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Print a warning message, uses logger.warning.
|
||||
"""
|
||||
msg, args, kwargs = self.process(msg, args, kwargs)
|
||||
|
||||
self.logger.warning(msg, *args, **kwargs)
|
||||
|
||||
def error(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Print an error message, uses logger.error.
|
||||
"""
|
||||
msg, args, kwargs = self.process(msg, args, kwargs)
|
||||
|
||||
self.logger.error(msg, *args, **kwargs)
|
||||
|
||||
def critical(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Print a critical message, uses logger.critical.
|
||||
"""
|
||||
msg, args, kwargs = self.process(msg, args, kwargs)
|
||||
|
||||
self.logger.critical(msg, *args, **kwargs)
|
||||
|
||||
def get_ip(self, ip):
|
||||
"""
|
||||
Lookup the assigned random IP for a given real IP.
|
||||
If no random IP exists, a new one is created and a message is logged indicating it.
|
||||
"""
|
||||
if ip not in self.ips:
|
||||
random_ip = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
|
||||
self.logger.info("Registering new random IP: %s" % random_ip)
|
||||
self.ips[ip] = random_ip
|
||||
|
||||
def process(self, msg, args, kwargs):
|
||||
"""
|
||||
Modify the log message to replace any instance of an IP in msg or args with its assigned random IP.
|
||||
"""
|
||||
new_args = []
|
||||
for arg in args:
|
||||
if type(arg) == str:
|
||||
for ip in self.regex.findall(arg):
|
||||
new_ip = self.get_ip(ip)
|
||||
|
||||
arg = arg.replace(ip, self.ips[ip])
|
||||
|
||||
new_args.append(arg)
|
||||
|
||||
for ip in self.regex.findall(msg):
|
||||
if ip not in self.ips:
|
||||
random_ip = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
|
||||
self.logger.debug("Registering new random IP: %s" % random_ip)
|
||||
self.ips[ip] = random_ip
|
||||
new_ip = self.get_ip(ip)
|
||||
|
||||
msg = msg.replace(ip, self.ips[ip])
|
||||
|
||||
return msg, tuple(new_args), kwargs
|
||||
|
||||
def close_logger(logger):
|
||||
"""
|
||||
|
@ -135,6 +242,34 @@ def close_logger(logger):
|
|||
handler.close()
|
||||
|
||||
|
||||
class Logger():
|
||||
"""
|
||||
Logging class context manager, as a thin wrapper around the logging class to help
|
||||
handle closing open file descriptors.
|
||||
"""
|
||||
def __init__(self, log_dir, logger_name, log_name, environment_id, log_level="DEBUG"):
|
||||
self.log_dir = log_dir
|
||||
self.logger_name = logger_name
|
||||
self.log_name = log_name
|
||||
self.environment_id = environment_id
|
||||
self.log_level = log_level
|
||||
self.logger = None
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
Sets up a logger.
|
||||
"""
|
||||
self.logger = get_logger(PROJECT_ROOT, self.log_dir, self.logger_name, self.log_name, self.environment_id, log_level=self.log_level)
|
||||
return self.logger
|
||||
|
||||
def __exit__(self, exc_type, exc_value, tb):
|
||||
"""
|
||||
Closes file handles.
|
||||
"""
|
||||
close_logger(self.logger)
|
||||
|
||||
|
||||
|
||||
def get_console_log_level():
|
||||
"""
|
||||
returns log level of console handler
|
||||
|
@ -142,6 +277,84 @@ def get_console_log_level():
|
|||
return CONSOLE_LOG_LEVEL
|
||||
|
||||
|
||||
def get_plugins():
|
||||
"""
|
||||
Iterates over this current directory to retrieve plugins.
|
||||
"""
|
||||
plugins = []
|
||||
for f in os.listdir(os.path.join(PROJECT_ROOT, "plugins")):
|
||||
if os.path.isdir(os.path.join(PROJECT_ROOT, "plugins", f)) and "__pycache__" not in f:
|
||||
plugins.append(f)
|
||||
return plugins
|
||||
|
||||
|
||||
def import_plugin(plugin, side):
|
||||
"""
|
||||
Imports given plugin.
|
||||
Args:
|
||||
- plugin: plugin to import (e.g. "http")
|
||||
- side: which side of the connection should be imported ("client" or "server")
|
||||
"""
|
||||
|
||||
# Define the full module for this plugin
|
||||
mod = "plugins.%s.%s" % (plugin, side)
|
||||
|
||||
path = os.path.join(PROJECT_ROOT, "plugins", plugin)
|
||||
if path not in sys.path:
|
||||
sys.path.append(path)
|
||||
|
||||
# Import the module
|
||||
importlib.import_module(mod)
|
||||
|
||||
# Predicate to filter classmembers
|
||||
def check_plugin(obj):
|
||||
"""
|
||||
Filters class members to ensure we get only enabled Plugin subclasses
|
||||
"""
|
||||
return inspect.isclass(obj) and \
|
||||
issubclass(obj, plugins.plugin.Plugin) and \
|
||||
(obj != plugins.plugin_client.ClientPlugin and \
|
||||
obj != plugins.plugin_server.ServerPlugin and \
|
||||
obj != plugins.plugin.Plugin) and \
|
||||
obj(None).enabled
|
||||
|
||||
# Filter the class members of the imported module to find our Plugin subclass
|
||||
clsmembers = inspect.getmembers(sys.modules[mod], predicate=check_plugin)
|
||||
|
||||
# Sanity check the class members we identified
|
||||
assert clsmembers, "Could not find plugin %s" % mod
|
||||
assert len(clsmembers) == 1, "Too many matching plugins found for %s" % mod
|
||||
|
||||
# Extract the class - clsmembers[0] is a tuple of (name, class)
|
||||
_, cls = clsmembers[0]
|
||||
|
||||
# Return the module path and class
|
||||
return mod, cls
|
||||
|
||||
|
||||
def build_command(args):
|
||||
"""
|
||||
Given a dictionary of arguments, build it back into a command line string.
|
||||
"""
|
||||
cmd = []
|
||||
for opt in args:
|
||||
# Don't pass along store true args that are false
|
||||
if args[opt] in [False, None]:
|
||||
continue
|
||||
cmd.append("--%s" % opt.replace("_", "-"))
|
||||
# If store true arg, we don't need to pass the value
|
||||
if args[opt] is True:
|
||||
continue
|
||||
|
||||
if args[opt] is '':
|
||||
cmd.append("''")
|
||||
elif " " in str(args[opt]):
|
||||
cmd.append("\"" + str(args[opt]) + "\"")
|
||||
else:
|
||||
cmd.append(str(args[opt]))
|
||||
return cmd
|
||||
|
||||
|
||||
def string_to_protocol(protocol):
|
||||
"""
|
||||
Converts string representations of scapy protocol objects to
|
||||
|
@ -177,6 +390,103 @@ def setup_dirs(output_dir):
|
|||
return ga_log_dir
|
||||
|
||||
|
||||
def get_from_fuzzed_or_real_packet(environment_id, real_packet_probability, enable_options=True, enable_load=True):
|
||||
"""
|
||||
Retrieves a protocol, field, and value from a fuzzed or real packet, depending on
|
||||
the given probability and if given packets is not None.
|
||||
"""
|
||||
packets = actions.utils.read_packets(environment_id)
|
||||
if packets and random.random() < real_packet_probability:
|
||||
packet = random.choice(packets)
|
||||
return packet.get_random()
|
||||
return layers.packet.Packet().gen_random()
|
||||
|
||||
|
||||
def read_packets(environment_id):
|
||||
"""
|
||||
Reads the pcap file associated with the last evaluation of this strategy.
|
||||
Returns a list of Geneva Packet objects.
|
||||
"""
|
||||
if not environment_id:
|
||||
return None
|
||||
|
||||
packets_path = os.path.join(RUN_DIRECTORY, "packets", "original_" + str(environment_id) + ".pcap")
|
||||
if not os.path.exists(packets_path):
|
||||
return None
|
||||
|
||||
parsed = []
|
||||
try:
|
||||
packets = rdpcap(packets_path)
|
||||
parsed = [layers.packet.Packet(p) for p in packets]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("FAILED TO PARSE!")
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def punish_fitness(fitness, logger, eng):
|
||||
"""
|
||||
Adjusts fitness based on additional optimizer functions.
|
||||
"""
|
||||
if not eng:
|
||||
logger.warning("Requested fitness adjustment without an engine - returning original fitness.")
|
||||
return fitness
|
||||
logger.debug("Initiating fitness adjustment")
|
||||
if eng and eng.strategy:
|
||||
fitness = punish_complexity(fitness, logger, eng.strategy)
|
||||
fitness = punish_unused(fitness, logger, eng.strategy)
|
||||
if fitness > 0:
|
||||
overhead = int(eng.overhead / 2)
|
||||
logger.debug("Punishing for overhead: %d" % overhead)
|
||||
fitness -= overhead
|
||||
|
||||
return fitness
|
||||
|
||||
|
||||
def punish_unused(fitness, logger, ind):
|
||||
"""
|
||||
Punishes strategy for each action that was not run.
|
||||
"""
|
||||
if not ind:
|
||||
return fitness
|
||||
logger.debug("Punishing for unused actions")
|
||||
num_unused = [action_tree.ran for action_tree in ind.out_actions].count(False)
|
||||
fitness -= (num_unused * 10)
|
||||
logger.debug(" - Number of unused actions in out forest: %d" % num_unused)
|
||||
num_unused = [action_tree.ran for action_tree in ind.in_actions].count(False)
|
||||
fitness -= (num_unused * 10)
|
||||
logger.debug(" - Number of unused actions in in forest: %d" % num_unused)
|
||||
return fitness
|
||||
|
||||
|
||||
def punish_complexity(fitness, logger, ind):
|
||||
"""
|
||||
Reduces fitness based on number of actions - optimizes for simplicity.
|
||||
"""
|
||||
if not ind:
|
||||
return fitness
|
||||
# Punish for number of actions
|
||||
if fitness > 0:
|
||||
logger.debug("Punishing for complexity: %d" % len(ind))
|
||||
fitness -= len(ind)
|
||||
return fitness
|
||||
|
||||
|
||||
def write_fitness(fitness, output_path, eid):
|
||||
"""
|
||||
Writes fitness to disk.
|
||||
"""
|
||||
try:
|
||||
float(fitness)
|
||||
except ValueError:
|
||||
print("Given fitness (%r) is not a number!" % fitness)
|
||||
raise
|
||||
fitpath = os.path.join(PROJECT_ROOT, output_path, FLAGFOLDER, eid) + ".fitness"
|
||||
with open(fitpath, "w") as fitfile:
|
||||
fitfile.write(str(fitness))
|
||||
|
||||
|
||||
def get_interface():
|
||||
"""
|
||||
Chooses an interface on the machine to use for socket testing.
|
||||
|
@ -189,3 +499,26 @@ def get_interface():
|
|||
# Filter for IPv4 addresses
|
||||
if netifaces.AF_INET in info:
|
||||
return iface
|
||||
|
||||
|
||||
def get_worker(name, logger):
|
||||
"""
|
||||
Returns information dictionary about a worker given its name.
|
||||
"""
|
||||
path = os.path.join("workers", name, "worker.json")
|
||||
if os.path.exists(name):
|
||||
path = name
|
||||
|
||||
dirpath = os.path.dirname(path)
|
||||
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
with open(path, "r") as fd:
|
||||
data = json.load(fd)
|
||||
|
||||
# If there is a private key, update the path to be relative to the project base
|
||||
if data.get("keyfile"):
|
||||
data["keyfile"] = os.path.join(dirpath, data["keyfile"])
|
||||
|
||||
return data
|
||||
|
|
|
@ -0,0 +1,163 @@
|
|||
import socket
|
||||
socket.setdefaulttimeout(1)
|
||||
import logging
|
||||
import random
|
||||
import os
|
||||
|
||||
import layers.packet
|
||||
import actions.utils
|
||||
|
||||
# Squelch annoying scapy ::1 runtime errors
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
|
||||
# Netfilterqueue may not work outside the docker container,
|
||||
# but this file can still be imported outside the docker container
|
||||
try:
|
||||
from netfilterqueue import NetfilterQueue
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from scapy.all import send, IP
|
||||
|
||||
# Note that censor.py lives in censors, so we need an extra dirname() call to get
|
||||
# to the project root
|
||||
BASEPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
class Censor(object):
|
||||
def __init__(self, eid, log_dir, log_level, port, queue_num):
|
||||
"""
|
||||
Setup censor attributes and logging.
|
||||
"""
|
||||
self.enabled = True
|
||||
self.nfqueue = None
|
||||
self.running_nfqueue = False
|
||||
self.queue_num = queue_num
|
||||
self.port = port
|
||||
self.eid = eid
|
||||
self.logger = None
|
||||
self.log_dir = log_dir
|
||||
if log_level:
|
||||
self.logger = actions.utils.get_logger(BASEPATH, log_dir, __name__, "censor", eid, log_level=log_level)
|
||||
self.logger.debug("Censor created to port %d on queue %d" % (port, queue_num))
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
Initialize the censor.
|
||||
"""
|
||||
self.logger.debug("Censor initializing.")
|
||||
|
||||
# Set up iptables rules to catch packets
|
||||
os.system("iptables -A FORWARD -j NFQUEUE -p tcp --sport %s --queue-num %s" % (self.port, self.queue_num))
|
||||
os.system("iptables -A FORWARD -j NFQUEUE -p tcp --dport %s --queue-num %s" % (self.port, self.queue_num))
|
||||
self.logger.debug("Censor iptables added")
|
||||
|
||||
#self.running_nfqueue = True
|
||||
self.num = 0
|
||||
try:
|
||||
self.nfqueue = NetfilterQueue()
|
||||
self.logger.debug("Censor binding")
|
||||
self.nfqueue.bind(int(self.queue_num), self.callback)
|
||||
self.logger.debug("Censor bound")
|
||||
self.nfqueue.run()
|
||||
except KeyboardInterrupt:
|
||||
self.logger.debug("CENSOR GOT SHUTDOWN")
|
||||
self.shutdown()
|
||||
#self.nfqueue_socket = socket.fromfd(self.nfqueue.get_fd(), socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
#self.nfqueue_thread = threading.Thread(target=self.run_nfqueue)
|
||||
#self.nfqueue_thread.start()
|
||||
# Spin wait the main thread while running nfqueue in the other threads
|
||||
#while self.running_nfqueue:
|
||||
# time.sleep(1)
|
||||
|
||||
def check_exit(self):
|
||||
"""
|
||||
Check if a shutdown flag has been written.
|
||||
"""
|
||||
flag_folder = os.path.join(BASEPATH, self.log_dir, actions.utils.FLAGFOLDER)
|
||||
if not os.path.exists(flag_folder):
|
||||
os.makedirs(flag_folder)
|
||||
return os.path.exists(os.path.join(flag_folder, "shutdown"))
|
||||
|
||||
def run_nfqueue(self):
|
||||
"""
|
||||
Run nfqueue in a non-blocking way. Note that nfqueue reports
|
||||
that it supports non-blocking operation, but this is broken in the
|
||||
library, and the following is the workaround.
|
||||
"""
|
||||
try:
|
||||
while self.running_nfqueue:
|
||||
try:
|
||||
self.nfqueue.run_socket(self.nfqueue_socket)
|
||||
except socket.timeout:
|
||||
self.logger.debug("Exiting")
|
||||
# Check if we need to exit
|
||||
if self.check_exit():
|
||||
break
|
||||
pass
|
||||
self.shutdown()
|
||||
except Exception:
|
||||
self.logger.exception("Exception out of run_nfqueue()")
|
||||
|
||||
def mysend(self, packet):
|
||||
"""
|
||||
Sends a packet with scapy.
|
||||
"""
|
||||
if "TCP" in packet:
|
||||
self.logger.debug(layers.packet.Packet._str_packet(packet))
|
||||
send(packet, verbose=False)
|
||||
return
|
||||
|
||||
def get_payload(self, packet):
|
||||
"""
|
||||
Parse paylaod out of the given scapy packet.
|
||||
"""
|
||||
payload = bytes(packet["TCP"].payload)
|
||||
if str(payload) != "b''":
|
||||
return payload
|
||||
else:
|
||||
return b""
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shuts down and cleans up the censor.
|
||||
"""
|
||||
self.logger.debug("Shutting down censor.")
|
||||
self.running_nfqueue = False
|
||||
self.nfqueue.unbind()
|
||||
#self.nfqueue_socket.close()
|
||||
os.system("iptables -D FORWARD -j NFQUEUE -p tcp --sport %s --queue-num %s" % (self.port, self.queue_num))
|
||||
os.system("iptables -D FORWARD -j NFQUEUE -p tcp --dport %s --queue-num %s" % (self.port, self.queue_num))
|
||||
|
||||
def callback(self, packet):
|
||||
"""
|
||||
NFQueue bound callback to capture packets and check whether we
|
||||
want to censor it.
|
||||
"""
|
||||
try:
|
||||
scapy_packet = IP(packet.get_payload())
|
||||
# Check for control check packet from evaluator to announce readiness
|
||||
if scapy_packet.sport == 2222 and scapy_packet.seq == 13337:
|
||||
# This line cannot be removed - it is to signal to the client the censor is ready
|
||||
flag_folder = os.path.join(BASEPATH, self.log_dir, actions.utils.FLAGFOLDER)
|
||||
if not os.path.exists(flag_folder):
|
||||
os.makedirs(flag_folder)
|
||||
ready_path = os.path.join(flag_folder, "%s.censor_ready" % self.eid)
|
||||
self.logger.debug("Writing ready file to %s" % ready_path)
|
||||
if not os.path.exists(ready_path):
|
||||
os.system("touch %s" % ready_path)
|
||||
self.logger.debug("Censor ready.")
|
||||
packet.drop()
|
||||
return
|
||||
action = "accept"
|
||||
# Check if the packet should be censored
|
||||
if self.check_censor(scapy_packet):
|
||||
# If so, trigger the censoring itself (drop packet, send RST, etc)
|
||||
action = self.censor(scapy_packet)
|
||||
|
||||
if action == "drop":
|
||||
packet.drop()
|
||||
else:
|
||||
packet.accept()
|
||||
except Exception:
|
||||
self.logger.exception("Censor exception in nfqueue callback.")
|
|
@ -0,0 +1,86 @@
|
|||
"""
|
||||
Censor 1
|
||||
|
||||
Designed to be run by the evaluator.
|
||||
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time,
|
||||
drops all packets after a TCP forbidden keyword is detected.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import layers.packet
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor1(Censor):
|
||||
"""
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time,
|
||||
drops all packets after a TCP forbidden keyword is detected.
|
||||
"""
|
||||
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
self.num = 0
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Initial TCP synchronization
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S":
|
||||
self.tcb = packet["TCP"].seq + 1
|
||||
self.logger.debug(("Synchronizing TCB (%d) on packet " + layers.packet.Packet._str_packet(packet)) % self.tcb)
|
||||
return False
|
||||
|
||||
# If we're tracking this packet stream
|
||||
if packet["TCP"].seq == self.tcb:
|
||||
self.tcb += len(self.get_payload(packet))
|
||||
else:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Censor 1 exception caught")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
"""
|
||||
Censor 10 is a RST censor designed to more closely mimic GFW behavior. It
|
||||
tracks multiple connections using TCBs, and will enter a TCB resynchronization
|
||||
state if a RST or FIN is sent and the full tuple of the TCB matches (src, dst,
|
||||
sport, dport) an existing TCB. It creates new TCBs for connections it is not
|
||||
yet aware of, and it checks all checksums of incoming packets (and ignores those
|
||||
that are incorrect), meaning insertion packets with incorrect checksums will not
|
||||
work.
|
||||
"""
|
||||
|
||||
import netifaces
|
||||
import layers.packet
|
||||
from censors.censor import Censor
|
||||
from scapy.all import raw, IP, TCP
|
||||
|
||||
|
||||
class Censor10(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcbs = []
|
||||
self.flagged_ips = []
|
||||
self.resynchronize = {}
|
||||
self.censor_interfaces = netifaces.interfaces()
|
||||
if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces:
|
||||
self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
|
||||
|
||||
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet.
|
||||
Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
if packet["IP"].src in self.flagged_ips:
|
||||
self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Record the reported checksum for the incoming packet
|
||||
reported_chksum = packet["TCP"].chksum
|
||||
# Remove the checksum for the packet so we can recalculate it
|
||||
del packet["TCP"].chksum
|
||||
|
||||
# Note this is actually what scapy's show2 method does under the hood
|
||||
# if curious, (see packet.py in scapy for show2 details)
|
||||
calculated_chksum = packet.__class__(raw(packet))["TCP"].chksum
|
||||
if reported_chksum != calculated_chksum:
|
||||
self.logger.debug("Packet checksum (%d) is incorrect (correct=%d). Ignoring." % (reported_chksum, calculated_chksum))
|
||||
return False
|
||||
|
||||
# If we are in a resynchronization state, or we do not yet have a connection and a new one
|
||||
# is being created, add or update a TCB
|
||||
tcb = self.get_matching_tcb(packet)
|
||||
if (tcb and self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])]) or \
|
||||
(not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S"]):
|
||||
|
||||
# Check if we've been tracking a connection for this ip:port <-> ip:port already,
|
||||
# so we can just replace that tcb with updated info
|
||||
if not tcb:
|
||||
tcb = self.get_partial_tcb(packet)
|
||||
if tcb is None:
|
||||
self.logger.debug("Making a new TCB for packet %s" % layers.packet.Packet._str_packet(packet))
|
||||
tcb = {}
|
||||
|
||||
tcb["src"] = packet["IP"].src
|
||||
tcb["dst"] = packet["IP"].dst
|
||||
tcb["sport"] = packet["TCP"].sport
|
||||
tcb["dport"] = packet["TCP"].dport
|
||||
tcb["seq"] = packet["TCP"].seq
|
||||
# If we're synchronizing on a SYN flag, need to add 1.
|
||||
if packet["TCP"].sprintf('%TCP.flags%') in ["S"]:
|
||||
tcb["seq"] += 1
|
||||
else:
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
self.tcbs.append(tcb)
|
||||
self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = False
|
||||
self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
# If connection is getting torn down
|
||||
elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "F"]:
|
||||
self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = True
|
||||
self.logger.debug(("Entering resynchronization state on packet %s" % layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if not tcb:
|
||||
self.logger.debug("No TCB matches packet.")
|
||||
return False
|
||||
|
||||
# Keep the TCB up to date
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 10")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Adds client and server IPs to flagged IP list.
|
||||
"""
|
||||
if scapy_packet["IP"].src not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].src)
|
||||
self.logger.debug("Marking IP %s for censorship..." % scapy_packet["IP"].src)
|
||||
if scapy_packet["IP"].dst not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].dst)
|
||||
self.logger.debug("Marking IP %s for censorship..." % scapy_packet["IP"].dst)
|
||||
|
||||
client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)
|
||||
client_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].sport,
|
||||
sport=scapy_packet[TCP].dport,
|
||||
ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)),
|
||||
seq=scapy_packet[TCP].ack,
|
||||
flags="R"
|
||||
)
|
||||
client_rst = client_ip_rst / client_tcp_rst
|
||||
|
||||
server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst)
|
||||
server_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].dport,
|
||||
sport=scapy_packet[TCP].sport,
|
||||
ack=scapy_packet[TCP].ack,
|
||||
seq=scapy_packet[TCP].seq,
|
||||
flags="R"
|
||||
)
|
||||
server_tcp_rst.show()
|
||||
server_rst = server_ip_rst / server_tcp_rst
|
||||
|
||||
for _ in range(0, 5):
|
||||
self.mysend(client_rst)
|
||||
self.mysend(server_rst)
|
||||
|
||||
return "accept"
|
||||
|
||||
def get_matching_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches the stored TCB.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"] and \
|
||||
packet["TCP"].seq == tcb["seq"]):
|
||||
return tcb
|
||||
return None
|
||||
|
||||
def get_partial_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches an existing connection, regardless if the SEQ/ACK
|
||||
are correct.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"]):
|
||||
return tcb
|
||||
return None
|
|
@ -0,0 +1,176 @@
|
|||
"""
|
||||
Censor 11 is a RST censor designed to more closely mimic GFW behavior. It
|
||||
tracks multiple connections using TCBs, and will enter a TCB resynchronization
|
||||
state if a RST or FIN is sent and the full tuple of the TCB matches (src, dst,
|
||||
sport, dport) an existing TCB. It creates new TCBs for connections it is not
|
||||
yet aware of, and it checks all checksums of incoming packets (and ignores those
|
||||
that are incorrect), meaning insertion packets with incorrect checksums will not
|
||||
work. It also resynchronizes on both S and ACK, defeating strategies that trigger
|
||||
before the 3-way handshake has finished.
|
||||
"""
|
||||
|
||||
import layers.packet
|
||||
import netifaces
|
||||
from censors.censor import Censor
|
||||
from scapy.all import raw, IP, TCP
|
||||
|
||||
|
||||
class Censor11(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcbs = []
|
||||
self.flagged_ips = []
|
||||
self.resynchronize = {}
|
||||
self.censor_interfaces = netifaces.interfaces()
|
||||
if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces:
|
||||
self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
|
||||
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet.
|
||||
Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
if packet["IP"].src in self.flagged_ips:
|
||||
self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
if packet["TCP"].dataofs < 5:
|
||||
return False
|
||||
# Record the reported checksum for the incoming packet
|
||||
reported_chksum = packet["TCP"].chksum
|
||||
# Remove the checksum for the packet so we can recalculate it
|
||||
del packet["TCP"].chksum
|
||||
|
||||
# Note this is actually what scapy's show2 method does under the hood
|
||||
# if curious, (see packet.py in scapy for show2 details)
|
||||
calculated_chksum = packet.__class__(raw(packet))["TCP"].chksum
|
||||
if reported_chksum != calculated_chksum:
|
||||
self.logger.debug("Packet checksum (%d) is incorrect (correct=%d). Ignoring." % (reported_chksum, calculated_chksum))
|
||||
return False
|
||||
|
||||
# If we are in a resynchronization state, or we do not yet have a connection and a new one
|
||||
# is being created, add or update a TCB
|
||||
tcb = self.get_matching_tcb(packet)
|
||||
if (tcb and self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])]) or \
|
||||
(not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S", "A"]):
|
||||
|
||||
# Check if we've been tracking a connection for this ip:port <-> ip:port already,
|
||||
# so we can just replace that tcb with updated info
|
||||
tcb = self.get_partial_tcb(packet)
|
||||
if tcb is None:
|
||||
self.logger.debug("Making a new TCB for packet %s" % layers.packet.Packet._str_packet(packet))
|
||||
tcb = {}
|
||||
|
||||
tcb["src"] = packet["IP"].src
|
||||
tcb["dst"] = packet["IP"].dst
|
||||
tcb["sport"] = packet["TCP"].sport
|
||||
tcb["dport"] = packet["TCP"].dport
|
||||
tcb["seq"] = packet["TCP"].seq
|
||||
# If we're synchronizing on a SYN flag, need to add 1.
|
||||
if packet["TCP"].sprintf('%TCP.flags%') in ["S"]:
|
||||
tcb["seq"] += 1
|
||||
else:
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
self.tcbs.append(tcb)
|
||||
self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = False
|
||||
self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
# If connection is getting torn down
|
||||
elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "F", "RA", "FA"]:
|
||||
self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = True
|
||||
self.logger.debug(("Entering resynchronization state on packet %s" % layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if not tcb:
|
||||
self.logger.debug("No TCB matches packet.")
|
||||
return False
|
||||
|
||||
# Keep the TCB up to date
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 10")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Adds client and server IPs to flagged IP list.
|
||||
"""
|
||||
if scapy_packet["IP"].src not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].src)
|
||||
self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].src)
|
||||
if scapy_packet["IP"].dst not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].dst)
|
||||
self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].dst)
|
||||
|
||||
client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)
|
||||
client_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].sport,
|
||||
sport=scapy_packet[TCP].dport,
|
||||
ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)),
|
||||
seq=scapy_packet[TCP].ack,
|
||||
flags="R"
|
||||
)
|
||||
client_rst = client_ip_rst / client_tcp_rst
|
||||
|
||||
server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst)
|
||||
server_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].dport,
|
||||
sport=scapy_packet[TCP].sport,
|
||||
ack=scapy_packet[TCP].ack,
|
||||
seq=scapy_packet[TCP].seq,
|
||||
flags="R"
|
||||
)
|
||||
server_tcp_rst.show()
|
||||
server_rst = server_ip_rst / server_tcp_rst
|
||||
|
||||
for _ in range(0, 5):
|
||||
self.mysend(client_rst)
|
||||
self.mysend(server_rst)
|
||||
|
||||
return "accept"
|
||||
|
||||
def get_matching_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches the stored TCB.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"] and \
|
||||
packet["TCP"].seq == tcb["seq"]):
|
||||
return tcb
|
||||
return None
|
||||
|
||||
def get_partial_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches an existing connection, regardless if the SEQ/ACK
|
||||
are correct.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"]):
|
||||
return tcb
|
||||
return None
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
Censor 2 ----> CENSOR 1
|
||||
|
||||
Designed to be run by the evaluator.
|
||||
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to client.
|
||||
"""
|
||||
|
||||
import layers.packet
|
||||
import logging
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor2(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: %s", layers.packet.Packet._str_packet(packet))
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S":
|
||||
self.tcb = packet["TCP"].seq + 1
|
||||
self.logger.debug("Synchronizing TCB on packet: %s", layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
if packet["TCP"].seq == self.tcb:
|
||||
self.tcb += len(self.get_payload(packet))
|
||||
|
||||
else:
|
||||
self.logger.debug("Ignoring packet: %s", layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: %s", layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Censor 2 exception caught.")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Send 5 resets to the client.
|
||||
"""
|
||||
rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)/TCP(dport=scapy_packet[TCP].sport, sport=scapy_packet[TCP].dport, ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), seq=scapy_packet[TCP].ack, flags="R")
|
||||
for i in range(0, 5):
|
||||
self.mysend(rst)
|
|
@ -0,0 +1,98 @@
|
|||
"""
|
||||
Censor 3 ----> CENSOR 2
|
||||
|
||||
Designed to be run by the evaluator.
|
||||
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to
|
||||
server AND client.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import netifaces
|
||||
import layers.packet
|
||||
# Disable scapy ::1 warnings
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
|
||||
from scapy.all import IP, TCP, wrpcap
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor3(Censor):
|
||||
"""
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to
|
||||
server AND client.
|
||||
"""
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.enabled = True
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
self.num = 0
|
||||
self.censor_interfaces = netifaces.interfaces()
|
||||
if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces:
|
||||
self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.num += 1
|
||||
|
||||
# Only censor TCP packets for now
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S":
|
||||
self.tcb = packet["TCP"].seq + 1
|
||||
self.logger.debug("Synchronizing TCB on packet " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
if packet["TCP"].seq == self.tcb:
|
||||
self.tcb += len(self.get_payload(packet))
|
||||
|
||||
else:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Censor 3 Error caught.")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Send 5 resets to the client and the server.
|
||||
"""
|
||||
client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)
|
||||
client_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].sport,
|
||||
sport=scapy_packet[TCP].dport,
|
||||
ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)),
|
||||
seq=scapy_packet[TCP].ack,
|
||||
flags="R"
|
||||
)
|
||||
client_rst = client_ip_rst / client_tcp_rst
|
||||
|
||||
server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst)
|
||||
server_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].dport,
|
||||
sport=scapy_packet[TCP].sport,
|
||||
ack=scapy_packet[TCP].ack,
|
||||
seq=scapy_packet[TCP].seq,
|
||||
flags="R"
|
||||
)
|
||||
server_tcp_rst.show()
|
||||
server_rst = server_ip_rst / server_tcp_rst
|
||||
|
||||
for _ in range(0, 5):
|
||||
self.mysend(client_rst)
|
||||
self.mysend(server_rst)
|
|
@ -0,0 +1,86 @@
|
|||
"""
|
||||
Censor 4
|
||||
|
||||
Dropping censor that synchronizes TCB on all SYN and ACK packets.
|
||||
"""
|
||||
|
||||
import layers.packet
|
||||
import logging
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor4(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
self.num = 0
|
||||
|
||||
def check_censor(self, packet, verbose=False):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.num += 1
|
||||
|
||||
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Initial TCP synchronization
|
||||
if "S" == packet["TCP"].sprintf('%TCP.flags%'):
|
||||
self.tcb = packet["TCP"].seq + 1
|
||||
self.logger.debug(("Synchronizing TCB (%d) on S packet " + layers.packet.Packet._str_packet(packet)) % self.tcb)
|
||||
return False
|
||||
|
||||
if "A" == packet["TCP"].sprintf('%TCP.flags%'):
|
||||
self.tcb = packet["TCP"].seq
|
||||
self.logger.debug(("Synchronizing TCB (%d) on A packet " + layers.packet.Packet._str_packet(packet)) % self.tcb)
|
||||
return False
|
||||
|
||||
# If we're tracking this packet stream
|
||||
if packet["TCP"].seq == self.tcb:
|
||||
self.tcb += len(self.get_payload(packet))
|
||||
else:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by censor 4")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
"""
|
||||
Censor 5
|
||||
|
||||
Designed to be run by the evaluator.
|
||||
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to
|
||||
the server only.
|
||||
"""
|
||||
|
||||
import layers.packet
|
||||
import logging
|
||||
import netifaces
|
||||
# Disable scapy ::1 warnings
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
|
||||
from scapy.all import IP, TCP, wrpcap
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor5(Censor):
|
||||
"""
|
||||
TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to
|
||||
server AND client.
|
||||
"""
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.enabled = True
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
self.num = 0
|
||||
self.censor_interfaces = netifaces.interfaces()
|
||||
if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces:
|
||||
self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.num += 1
|
||||
|
||||
# Only censor TCP packets for now
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S":
|
||||
self.tcb = packet["TCP"].seq + 1
|
||||
self.logger.debug("Synchronizing TCB on packet " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
if packet["TCP"].seq == self.tcb:
|
||||
self.tcb += len(self.get_payload(packet))
|
||||
|
||||
else:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Censor 3 Error caught.")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Send 5 resets to the server.
|
||||
"""
|
||||
server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst)
|
||||
server_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].dport,
|
||||
sport=scapy_packet[TCP].sport,
|
||||
ack=scapy_packet[TCP].ack,
|
||||
seq=scapy_packet[TCP].seq,
|
||||
flags="R"
|
||||
)
|
||||
server_tcp_rst.show()
|
||||
server_rst = server_ip_rst / server_tcp_rst
|
||||
|
||||
for _ in range(0, 5):
|
||||
self.mysend(server_rst)
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
Censor 6 is a IP dropping TCB Teardown censor. It tears down the TCB on any
|
||||
FIN or RST packet.
|
||||
Does not check if the ports are correct for the FIN/RST.
|
||||
|
||||
"""
|
||||
import layers.packet
|
||||
import logging
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor6(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = 0
|
||||
self.drop_all_from = None
|
||||
|
||||
def check_censor(self, packet, verbose=False):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Some stacks send RA to tear down a connection
|
||||
if packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA", "F"]:
|
||||
self.tcb = None
|
||||
self.logger.debug(("Tearing down TCB on packet " + layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
if self.tcb is None:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 6")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Censor 7 is a IP dropping TCB Teardown censor. It only tears down a TCB
|
||||
if the full tuple of the TCB matches (src, dst, sport, dport).
|
||||
Does not check if the SEQ/ACK are in window for the FIN/RST.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import layers.packet
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor7(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = {}
|
||||
self.drop_all_from = None
|
||||
|
||||
def check_censor(self, packet, verbose=False):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Clients can close connections with any of these flags
|
||||
if packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA", "F"]:
|
||||
# If a TCB has already been setup, check that this packet matches it
|
||||
if self.tcb and \
|
||||
packet["IP"].src in self.tcb["ips"] and \
|
||||
packet["IP"].dst in self.tcb["ips"] and \
|
||||
packet["TCP"].dport in self.tcb["ports"] and \
|
||||
packet["TCP"].sport in self.tcb["ports"]:
|
||||
|
||||
self.tcb = None
|
||||
self.logger.debug(("Tearing down TCB on packet " + layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
elif not self.tcb and self.tcb is not None:
|
||||
self.tcb["ips"] = [packet["IP"].src, packet["IP"].dst]
|
||||
self.tcb["ports"] = [packet["TCP"].sport, packet["TCP"].dport]
|
||||
|
||||
if self.tcb is None:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 7")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
|
@ -0,0 +1,78 @@
|
|||
"""
|
||||
Censor 8 is a IP dropping TCB Teardown censor. It only tears down a TCB
|
||||
if the full tuple of the TCB matches (src, dst, sport, dport, seq).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import layers.packet
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor8(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = {}
|
||||
self.drop_all_from = None
|
||||
|
||||
def check_censor(self, packet, verbose=False):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S" and not self.tcb and self.tcb is not None:
|
||||
self.tcb["ips"] = [packet["IP"].src, packet["IP"].dst]
|
||||
self.tcb["ports"] = [packet["TCP"].sport, packet["TCP"].dport]
|
||||
self.tcb["seq"] = packet["TCP"].seq + 1
|
||||
self.logger.debug("Synchronizing TCB on packet " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
# TCB teardown
|
||||
elif packet["TCP"].sprintf('%TCP.flags%') == "R" or packet["TCP"].sprintf('%TCP.flags%') == "F":
|
||||
# If a TCB has already been setup, check that this packet matches it
|
||||
if self.tcb and \
|
||||
packet["IP"].src in self.tcb["ips"] and \
|
||||
packet["IP"].dst in self.tcb["ips"] and \
|
||||
packet["TCP"].dport in self.tcb["ports"] and \
|
||||
packet["TCP"].sport in self.tcb["ports"] and \
|
||||
packet["TCP"].seq == self.tcb["seq"]:
|
||||
|
||||
self.tcb = None
|
||||
self.logger.debug(("Tearing down TCB on packet " + layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
if self.tcb is None:
|
||||
self.logger.debug("Ignoring packet: " + layers.packet.Packet._str_packet(packet))
|
||||
return False
|
||||
elif "seq" in self.tcb and packet["TCP"].seq == self.tcb["seq"]:
|
||||
self.tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 8")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
|
@ -0,0 +1,159 @@
|
|||
"""
|
||||
Censor 8b is a RST censor designed to mimic TCB teardown GFW behavior. It
|
||||
tracks multiple connections using TCBs, but does not enter a TCB resynchronization
|
||||
state if a RST or FIN, it simply tears down. It creates new TCBs for connections it is not
|
||||
yet aware of, but does not check the checksums of incoming packets.
|
||||
"""
|
||||
|
||||
import layers.packet
|
||||
import netifaces
|
||||
from censors.censor import Censor
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
|
||||
class Censor8b(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcbs = []
|
||||
self.flagged_ips = []
|
||||
self.censor_interfaces = netifaces.interfaces()
|
||||
if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces:
|
||||
self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
|
||||
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet.
|
||||
Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
if packet["IP"].src in self.flagged_ips:
|
||||
self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# Throw away packets that have an invalid dataofs
|
||||
if packet["TCP"].dataofs < 5:
|
||||
return False
|
||||
|
||||
# If we are in a resynchronization state, or we do not yet have a connection and a new one
|
||||
# is being created, add or update a TCB
|
||||
tcb = self.get_matching_tcb(packet)
|
||||
if (not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S"]):
|
||||
# Check if we've been tracking a connection for this ip:port <-> ip:port already,
|
||||
# so we can just replace that tcb with updated info
|
||||
tcb = self.get_partial_tcb(packet)
|
||||
if tcb is None:
|
||||
self.logger.debug("Making a new TCB for packet %s" % layers.packet.Packet._str_packet(packet))
|
||||
tcb = {}
|
||||
|
||||
tcb["src"] = packet["IP"].src
|
||||
tcb["dst"] = packet["IP"].dst
|
||||
tcb["sport"] = packet["TCP"].sport
|
||||
tcb["dport"] = packet["TCP"].dport
|
||||
tcb["seq"] = packet["TCP"].seq
|
||||
# If we're synchronizing on a SYN flag, need to add 1.
|
||||
if packet["TCP"].sprintf('%TCP.flags%') in ["S"]:
|
||||
tcb["seq"] += 1
|
||||
else:
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
self.tcbs.append(tcb)
|
||||
self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
# If connection is getting torn down
|
||||
elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA"]:
|
||||
self.tcbs.remove(tcb)
|
||||
self.logger.debug(("Deleting TCB for packet %s" % layers.packet.Packet._str_packet(packet)))
|
||||
return False
|
||||
|
||||
if not tcb:
|
||||
self.logger.debug("No TCB matches packet.")
|
||||
return False
|
||||
|
||||
# Keep the TCB up to date
|
||||
tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: %s" % layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 8b")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Adds client and server IPs to flagged IP list.
|
||||
"""
|
||||
if scapy_packet["IP"].src not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].src)
|
||||
self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].src)
|
||||
if scapy_packet["IP"].dst not in self.flagged_ips:
|
||||
self.flagged_ips.append(scapy_packet["IP"].dst)
|
||||
self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].dst)
|
||||
|
||||
client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)
|
||||
client_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].sport,
|
||||
sport=scapy_packet[TCP].dport,
|
||||
ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)),
|
||||
seq=scapy_packet[TCP].ack,
|
||||
flags="R"
|
||||
)
|
||||
client_rst = client_ip_rst / client_tcp_rst
|
||||
|
||||
server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst)
|
||||
server_tcp_rst = TCP(
|
||||
dport=scapy_packet[TCP].dport,
|
||||
sport=scapy_packet[TCP].sport,
|
||||
ack=scapy_packet[TCP].ack,
|
||||
seq=scapy_packet[TCP].seq,
|
||||
flags="R"
|
||||
)
|
||||
server_tcp_rst.show()
|
||||
server_rst = server_ip_rst / server_tcp_rst
|
||||
|
||||
for _ in range(0, 5):
|
||||
self.mysend(client_rst)
|
||||
self.mysend(server_rst)
|
||||
|
||||
return "accept"
|
||||
|
||||
def get_matching_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches the stored TCB.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"] and \
|
||||
packet["TCP"].seq == tcb["seq"]):
|
||||
return tcb
|
||||
return None
|
||||
|
||||
def get_partial_tcb(self, packet):
|
||||
"""
|
||||
Checks if the packet matches an existing connection, regardless if the SEQ/ACK
|
||||
are correct.
|
||||
"""
|
||||
for tcb in self.tcbs:
|
||||
self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if (packet["IP"].src == tcb["src"] and \
|
||||
packet["IP"].dst == tcb["dst"] and \
|
||||
packet["TCP"].sport == tcb["sport"] and \
|
||||
packet["TCP"].dport == tcb["dport"]):
|
||||
return tcb
|
||||
return None
|
|
@ -0,0 +1,106 @@
|
|||
"""
|
||||
Censor 9 is a IP dropping TCB Teardown censor. It does not tear down its TCB,
|
||||
but it will resynchronize it's TCB if a RST or FIN is sent if the full tuple
|
||||
of the TCB matches (src, dst, sport, dport, seq).
|
||||
|
||||
More closely mimics GFW behavior.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import layers.packet
|
||||
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
|
||||
from scapy.all import IP, TCP
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Censor9(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
self.forbidden = forbidden
|
||||
self.tcb = {}
|
||||
self.drop_all_from = None
|
||||
self.resynchronize = False
|
||||
|
||||
def check_censor(self, packet, verbose=False):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns true or false.
|
||||
"""
|
||||
try:
|
||||
self.logger.debug("Inbound packet to censor: " + layers.packet.Packet._str_packet(packet))
|
||||
if self.drop_all_from == packet["IP"].src:
|
||||
self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from)
|
||||
return True
|
||||
|
||||
# Only censor TCP packets for now
|
||||
if "TCP" not in packet:
|
||||
return False
|
||||
|
||||
# If we are in a resynchronization state, or we do not yet have a connection and a new one
|
||||
# is being created, definte the TCB
|
||||
if self.resynchronize or (not self.tcb and packet["TCP"].sprintf('%TCP.flags%') == "S"):
|
||||
self.tcb["src"] = packet["IP"].src
|
||||
self.tcb["dst"] = packet["IP"].dst
|
||||
self.tcb["sport"] = packet["TCP"].sport
|
||||
self.tcb["dport"] = packet["TCP"].dport
|
||||
self.tcb["seq"] = packet["TCP"].seq
|
||||
# If we're synchronizing on a SYN flag, need to add 1.
|
||||
if packet["TCP"].sprintf('%TCP.flags%') == "S":
|
||||
self.tcb["seq"] += 1
|
||||
else:
|
||||
self.tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
self.resynchronize = False
|
||||
self.logger.debug("Synchronizing TCB on packet " + layers.packet.Packet._str_packet(packet))
|
||||
return self.check_forbidden(packet)
|
||||
|
||||
# If connection is getting torn down
|
||||
elif self.tcb_matches(packet) and \
|
||||
(packet["TCP"].sprintf('%TCP.flags%') == "R" or \
|
||||
packet["TCP"].sprintf('%TCP.flags%') == "F"):
|
||||
self.resynchronize = True
|
||||
self.logger.debug(("Entering resynchronization state on packet " + layers.packet.Packet._str_packet(packet)))
|
||||
|
||||
if not self.tcb_matches(packet):
|
||||
self.logger.debug("TCB does not match packet.")
|
||||
return False
|
||||
# Keep the TCB up to date
|
||||
elif "seq" in self.tcb:
|
||||
self.tcb["seq"] += len(self.get_payload(packet))
|
||||
|
||||
return self.check_forbidden(packet)
|
||||
|
||||
except Exception:
|
||||
self.logger.exception("Exception caught by Censor 9")
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Marks this IP to be dropped in the future and drops this packet.
|
||||
"""
|
||||
self.drop_all_from = scapy_packet["IP"].src
|
||||
self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from)
|
||||
return "drop"
|
||||
|
||||
def check_forbidden(self, packet):
|
||||
"""
|
||||
Checks if a packet contains forbidden words.
|
||||
"""
|
||||
# Check if any forbidden words appear in the packet payload
|
||||
for keyword in self.forbidden:
|
||||
if keyword in self.get_payload(packet):
|
||||
self.logger.debug("Packet triggered censor: " + layers.packet.Packet._str_packet(packet))
|
||||
return True
|
||||
return False
|
||||
|
||||
def tcb_matches(self, packet):
|
||||
"""
|
||||
Checks if the packet matches the stored TCB.
|
||||
"""
|
||||
self.logger.debug(self.tcb)
|
||||
return not self.tcb or (self.tcb and \
|
||||
packet["IP"].src == self.tcb["src"] and \
|
||||
packet["IP"].dst == self.tcb["dst"] and \
|
||||
packet["TCP"].sport == self.tcb["sport"] and \
|
||||
packet["TCP"].dport == self.tcb["dport"] and \
|
||||
packet["TCP"].seq == self.tcb["seq"])
|
|
@ -0,0 +1,94 @@
|
|||
import argparse
|
||||
import importlib
|
||||
import inspect
|
||||
import os
|
||||
import traceback
|
||||
import sys
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(BASEPATH)
|
||||
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.append(PROJECT_ROOT)
|
||||
|
||||
import censors.censor
|
||||
|
||||
CENSORS = {}
|
||||
|
||||
def get_censors():
|
||||
"""
|
||||
Dynamically imports all of the Censors classes in this directory.
|
||||
"""
|
||||
global CENSORS
|
||||
if CENSORS:
|
||||
return CENSORS
|
||||
|
||||
collected_censors = {}
|
||||
for censor_file in os.listdir(os.path.dirname(os.path.abspath(__file__))):
|
||||
if not censor_file.endswith(".py"):
|
||||
continue
|
||||
censor_file = censor_file.replace(".py", "")
|
||||
importlib.import_module("censors."+censor_file)
|
||||
def check_censor(o):
|
||||
return inspect.isclass(o) and issubclass(o, censors.censor.Censor) and o != censors.censor.Censor
|
||||
clsmembers = inspect.getmembers(sys.modules["censors."+censor_file], predicate=check_censor)
|
||||
if clsmembers:
|
||||
name, censor_class = clsmembers[0]
|
||||
if censor_class(0, [], None, None, None, None).enabled:
|
||||
collected_censors[name.lower()] = censor_class
|
||||
|
||||
CENSORS = collected_censors
|
||||
return collected_censors
|
||||
|
||||
|
||||
def get_args():
|
||||
"""
|
||||
Sets up argparse and collects arguments.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description='The server, run by the evaluator.')
|
||||
parser.add_argument('--port', type=int, action='store', help="Server port",
|
||||
required=True)
|
||||
parser.add_argument('--queue', type=int, action='store', help="NFQueue number to use",
|
||||
required=True)
|
||||
parser.add_argument('--environment-id', action='store', help="ID of the current environment",
|
||||
required=True)
|
||||
parser.add_argument('--censor', action='store', help="censor to deploy", required=True)
|
||||
parser.add_argument('--forbidden', action='store', default='ultrasurf', help="word to censor")
|
||||
parser.add_argument('--output-directory', action='store', help="Where to write logs",
|
||||
required=True)
|
||||
parser.add_argument('--log', action='store', default="debug",
|
||||
choices=("debug", "info", "warning", "critical", "error"),
|
||||
help="Sets the log level")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main(args):
|
||||
"""
|
||||
Starts the given censor.
|
||||
"""
|
||||
try:
|
||||
censors = get_censors()
|
||||
censor_name = args["censor"].lower()
|
||||
|
||||
if censor_name not in censors:
|
||||
print("ERROR: Unknown censor.")
|
||||
return None
|
||||
censor_cls = censors[censor_name]
|
||||
censor = censor_cls(args["environment_id"],
|
||||
[args["forbidden"].encode('utf-8')],
|
||||
args["output_directory"],
|
||||
args["log"],
|
||||
args["port"],
|
||||
args["queue"])
|
||||
print("Censor %s starting." % censor_name)
|
||||
censor.start()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# Note that this code can be removed - this is how the
|
||||
# evaluator runs the censor for tests in the Docker environment
|
||||
if __name__ == "__main__":
|
||||
main(vars(get_args()))
|
|
@ -0,0 +1,22 @@
|
|||
1. TCP Censor that synchronizes on first SYN only, works 100% of the time, drops all packets after TCP forbidden content
|
||||
- [flag:S]dup-tamper{TCP:syn:corrupt}
|
||||
2. TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to client after TCP forbidden content
|
||||
- [flag:R]drop OR [flag:S]dup-tamper{TCP:syn:corrupt}
|
||||
3. TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to server AND client after TCP forbidden content
|
||||
- [flag:S]dup-tamper{TCP:syn:corrupt}
|
||||
4. TCP Censor that synchronizes on first SYN, ACK only, works 100% of the time, drops all packets after TCP forbidden content
|
||||
- [flag:S,A]dup-tamper{TCP:syn:corrupt}
|
||||
5. TCP Censor that synchronizes on first SYN and every 5 packets after that
|
||||
- [flag:S]dup-tamper{TCP:syn:corrupt},[cyclic:4]dup-tamper{TCP:syn:corrupt}
|
||||
6. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached only to client/server IP is not torn down
|
||||
7. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached to client/server IP and port is not torn down
|
||||
8. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ is not torn down
|
||||
- [field:TCP:flags:A]-duplicate-tamper{TCP:flags:replace:R}-tamper{TCP:chksum:replace:4444}-| \/
|
||||
9. TCP Censor that works 100% of the time, tracks only 1 connection, drops all packets after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ if TCB matches
|
||||
- Need to force re-sync w/ FIN/RST and then send insertion packet
|
||||
10. TCP Censor that can track multiple connections, can resynchronize its TCBs, checks packet checksums, works 100% of the time, drops all packets from either IP address after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ if TCB matches
|
||||
- Need to force re-sync w/ FIN/RST and then send insertion packet with a correct chksum
|
||||
|
||||
UDP:
|
||||
1. Censor that can inject responses to DNS requests
|
||||
- [default value in packet]drop
|
|
@ -0,0 +1,26 @@
|
|||
"""
|
||||
Dummy
|
||||
|
||||
Designed to be run by the evaluator.
|
||||
|
||||
Censors nothing - dummy censor for infrastructure testing.
|
||||
"""
|
||||
|
||||
from censors.censor import Censor
|
||||
|
||||
|
||||
class Dummy(Censor):
|
||||
def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num):
|
||||
Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num)
|
||||
|
||||
def check_censor(self, packet):
|
||||
"""
|
||||
Check if the censor should run against this packet. Returns False for dummy censor.
|
||||
"""
|
||||
return False
|
||||
|
||||
def censor(self, scapy_packet):
|
||||
"""
|
||||
Does nothing.
|
||||
"""
|
||||
return False
|
|
@ -0,0 +1,17 @@
|
|||
FROM python:3.6-stretch
|
||||
# Set the root password in the container so we can use it
|
||||
RUN echo "root:Docker!" | chpasswd
|
||||
RUN apt-get -y update
|
||||
RUN apt-get -y install libnetfilter-queue-dev iptables tcpdump netcat net-tools git graphviz openssh-server
|
||||
# Enable root SSH login for client testing
|
||||
RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
|
||||
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
|
||||
RUN sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config
|
||||
|
||||
# SSH login fix. Otherwise user is kicked off after login
|
||||
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
|
||||
|
||||
RUN DEBIAN_FRONTEND=noninteractive apt-get -y install tshark
|
||||
ENV PATH="/usr/sbin:${PATH}"
|
||||
RUN pip install netfilterqueue requests dnspython anytree graphviz netifaces paramiko tld docker scapy==2.4.3 psutil
|
||||
ENTRYPOINT ["/bin/bash"]
|
|
@ -0,0 +1,23 @@
|
|||
# Geneva Docker
|
||||
|
||||
This implements the Docker base image for Geneva. Each docker container used by the evaluator runs out of the same base container.
|
||||
|
||||
To build it:
|
||||
```
|
||||
docker build -t base:latest -f docker/Dockerfile .
|
||||
```
|
||||
|
||||
Optionally, to manually run/inspect the docker image to explore the image, run:
|
||||
|
||||
```
|
||||
docker run -it base
|
||||
```
|
||||
|
||||
You can run the base image with the below python:
|
||||
```python
|
||||
import os
|
||||
import docker
|
||||
|
||||
docker_client = docker.from_env()
|
||||
docker_client.containers.run('base', detach=True, privileged=True, volumes={os.path.abspath(os.getcwd()): {"bind" : "/code", "mode" : "rw"}}, tty=True, remove=True, name="test")
|
||||
```
|
|
@ -0,0 +1,20 @@
|
|||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.action
|
||||
=====================
|
||||
|
||||
.. automodule:: action
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.drop
|
||||
===================
|
||||
|
||||
.. automodule:: drop
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.duplicate
|
||||
========================
|
||||
|
||||
.. automodule:: duplicate
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.fragment
|
||||
=======================
|
||||
|
||||
.. automodule:: fragment
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.layers.layer
|
||||
====================
|
||||
|
||||
.. automodule:: layer
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.layers.packet
|
||||
=====================
|
||||
|
||||
.. automodule:: packet
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.sleep
|
||||
====================
|
||||
|
||||
.. automodule:: sleep
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.strategy
|
||||
=======================
|
||||
|
||||
.. automodule:: strategy
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.tamper
|
||||
=====================
|
||||
|
||||
.. automodule:: tamper
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.trace
|
||||
====================
|
||||
|
||||
.. automodule:: trace
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.tree
|
||||
====================
|
||||
|
||||
.. automodule:: tree
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.trigger
|
||||
======================
|
||||
|
||||
.. automodule:: trigger
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.actions.utils
|
||||
====================
|
||||
|
||||
.. automodule:: utils
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.engine
|
||||
=============
|
||||
|
||||
.. automodule:: engine
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.evaluator
|
||||
=================
|
||||
|
||||
.. automodule:: evaluator
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.evolve
|
||||
=============
|
||||
|
||||
.. automodule:: evolve
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,17 @@
|
|||
geneva.plugins.dns
|
||||
============================
|
||||
|
||||
.. automodule:: plugins.dns.client
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
.. automodule:: plugins.dns.server
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
.. automodule:: plugins.dns.plugin
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,12 @@
|
|||
geneva.plugins.echo
|
||||
============================
|
||||
|
||||
.. automodule:: plugins.echo.client
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
.. automodule:: plugins.echo.server
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,17 @@
|
|||
geneva.plugins.http
|
||||
============================
|
||||
|
||||
.. automodule:: plugins.http.client
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
.. automodule:: plugins.http.server
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
.. automodule:: plugins.http.plugin
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.plugins.plugin
|
||||
=====================
|
||||
|
||||
.. automodule:: plugin
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.plugins.plugin_client
|
||||
============================
|
||||
|
||||
.. automodule:: plugin_client
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.plugins.plugin_server
|
||||
============================
|
||||
|
||||
.. automodule:: plugin_server
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,7 @@
|
|||
geneva.plugins.sni
|
||||
============================
|
||||
|
||||
.. automodule:: plugins.sni.client
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
|
@ -0,0 +1,80 @@
|
|||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# This file only contains a selection of the most common options. For a full
|
||||
# list see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Path setup --------------------------------------------------------------
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.abspath('.'))
|
||||
sys.path.insert(1, os.path.dirname(os.path.abspath('.')))
|
||||
sys.path.insert(2, os.path.join(os.path.dirname(os.path.abspath('.')), "actions"))
|
||||
sys.path.insert(3, os.path.join(os.path.dirname(os.path.abspath('.')), "plugins"))
|
||||
# Hack so the HTTP plugin will import
|
||||
sys.path.insert(4, os.path.join(os.path.dirname(os.path.abspath('.')), "plugins", "http"))
|
||||
import sphinx_rtd_theme
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
||||
project = 'geneva'
|
||||
copyright = '2020, Kevin Bock'
|
||||
author = 'Kevin Bock'
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"sphinx_rtd_theme",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.autosectionlabel"
|
||||
]
|
||||
|
||||
master_doc = 'index'
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
autodoc_member_order = 'groupwise'
|
||||
|
||||
# Don't skip __init__
|
||||
def skip(app, what, name, obj, would_skip, options):
|
||||
if name == "__init__":
|
||||
return False
|
||||
return would_skip
|
||||
|
||||
def setup(app):
|
||||
app.connect("autodoc-skip-member", skip)
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
pygments_style = 'colorful'
|
||||
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'] }
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
Defining New Actions
|
||||
=====================
|
||||
|
||||
It is simple to add a new packet-level action to Geneva.
|
||||
|
||||
Let us assume we are adding a new action, called "mytamper", which simply sets the :code:`ipid` field of a packet.
|
||||
Our action will take 1 packet and return 1 packet, and we'll start by making it always set the IPID to 1.
|
||||
|
||||
We will subclasss the :code:`Action` class, and specify an :code:`__init__` method and a :code:`run` method.
|
||||
|
||||
In the :code:`__init__` method, we will specify that our action's name is 'mytamper' and can run in :code:`both` inbound and outbound.
|
||||
Then, in the :code:`run()` method, we will use Geneva's packet API to set the :code:`ipid` field to 1 and simply return the packet.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from actions.action import Action
|
||||
|
||||
class MyTamperAction(Action):
|
||||
"""
|
||||
Geneva action to set the IPID to 1.
|
||||
"""
|
||||
# Controls frequency with which this action is chosen by the genetic algorithm
|
||||
# during mutation
|
||||
frequency = 0
|
||||
|
||||
def __init__(self, environment_id=None):
|
||||
Action.__init__(self, "mytamper", "both")
|
||||
|
||||
def run(self, packet, logger):
|
||||
"""
|
||||
The mytamper action returns a modified packet as the left child.
|
||||
"""
|
||||
logger.debug(" - Changing IPID field to 1")
|
||||
packet.set("IP", "ipid", 1)
|
||||
return packet, None
|
||||
|
||||
And that's it! Now, we can specify this action in our normal strategy DNA: Geneva will discover it dynamically on startup, import it, and we can use it.
|
||||
|
||||
Adding Parameters
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
Let's now assume we want to make our action take parameters. We will add two new methods: :code:`parse()` and :code:`__str__()`.
|
||||
We'll start by adding a new instance variable :code:`self.ipid_value`.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def __init__(self, environment_id=None, ipid_value=1):
|
||||
Action.__init__(self, "mytamper", "both")
|
||||
self.ipid_value = ipid_value
|
||||
|
||||
Next, we'll add the :code:`__str__` method so when our action is printed in the strategy DNA, its components are too:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def __str__(self):
|
||||
"""
|
||||
Returns a string representation.
|
||||
"""
|
||||
s = Action.__str__(self)
|
||||
s += "{%g}" % self.ipid_value
|
||||
return s
|
||||
|
||||
Finally, we'll add the :code:`parse()` method so we can parse the value from a string strategy DNA to a live action.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def parse(self, string, logger):
|
||||
"""
|
||||
Parses a string representation for this object.
|
||||
"""
|
||||
try:
|
||||
if string:
|
||||
self.ipid_value = float(string)
|
||||
except ValueError:
|
||||
logger.exception("Cannot parse ipid_value %s" % string)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
Putting it all together:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from actions.action import Action
|
||||
|
||||
class MyTamperAction(Action):
|
||||
"""
|
||||
Geneva action to set the IPID to 1.
|
||||
"""
|
||||
# Controls frequency with which this action is chosen by the genetic algorithm
|
||||
# during mutation
|
||||
frequency = 0
|
||||
|
||||
def __init__(self, environment_id=None, ipid_value=1):
|
||||
Action.__init__(self, "mytamper", "both")
|
||||
self.ipid_value = ipid_value
|
||||
|
||||
def run(self, packet, logger):
|
||||
"""
|
||||
The mytamper action returns a modified packet as the left child.
|
||||
"""
|
||||
logger.debug(" - Changing IPID field to 1")
|
||||
packet.set("IP", "ipid", 1)
|
||||
return packet, None
|
||||
|
||||
def __str__(self):
|
||||
"""
|
||||
Returns a string representation.
|
||||
"""
|
||||
s = Action.__str__(self)
|
||||
s += "{%g}" % self.ipid_value
|
||||
return s
|
||||
|
||||
def parse(self, string, logger):
|
||||
"""
|
||||
Parses a string representation for this object.
|
||||
"""
|
||||
try:
|
||||
if string:
|
||||
self.ipid_value = float(string)
|
||||
except ValueError:
|
||||
logger.exception("Cannot parse ipid_value %s" % string)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
And we're done! Now, we can write strategies like: :code:`[TCP:flags:PA]-mytamper{10}-|`, and any TCP packet with the flags field set to :code:`PA` will have its :code:`ipid` field set to 10.
|
|
@ -0,0 +1,6 @@
|
|||
Contributing
|
||||
=============
|
||||
|
||||
Contributions are welcome! You are encouraged to fork the repository, open Github issues with us,
|
||||
or just make a pull request. If you are interested in becoming more involved with the team or
|
||||
development, checkout our website a `https://censorship.ai <https://censorship.ai>`_ and drop us a line!
|
|
@ -0,0 +1,2 @@
|
|||
Exposing New Protocols
|
||||
======================
|
|
@ -0,0 +1,276 @@
|
|||
Adding New Plugins
|
||||
==================
|
||||
|
||||
This section will describe the process to add a new application plugin to Geneva.
|
||||
Application plugins serve as the fitness function for Geneva during evolution,
|
||||
and allow it to evolve strategies to defeat certain types of censorship.
|
||||
|
||||
Plugins are run by the Evaluator; if you have not yet read how the Evaluator
|
||||
works, see the :ref:`Strategy Evaluation` section.
|
||||
|
||||
There are three types of plugins: clients, servers, and overriding plugins.
|
||||
A developer can choose to implement any one of, or all three of these plugins.
|
||||
|
||||
For this section, we will build an example plugin and walk through the existing
|
||||
plugins to tour through the plugin API.
|
||||
|
||||
Plugins are expected to be in the :code:`plugins/` folder in geneva's repo. The
|
||||
folder name is the plugin name, and Geneva will discover these automatically.
|
||||
Plugins are specified to the evaluator or evolve with the :code:`--test-type`
|
||||
flag. Within the plugin folder, plugins must adhere to the following naming
|
||||
scheme:
|
||||
|
||||
- :code:`client.py` - for plugin clients
|
||||
- :code:`server.py` - for plugin servers [optional]
|
||||
- :code:`plugin.py` - for an overriding plugin to customize logic [optional]
|
||||
|
||||
:code:`server.py` and :code:`plugin.py` are optional. The server plugin is
|
||||
required to do server-side evaluation, but the overriding plugin definition is
|
||||
only required to if a developer wishes to override the evaluator's default
|
||||
behavior.
|
||||
|
||||
If an overriding plugin is provided, the evaluator will simply invoke it at the
|
||||
start of strategy evaluation and the overriding plugin will be responsible for
|
||||
calling the client and server. This section will assume that no overriding
|
||||
plugin is specified to describe the evaluator's default behavior with plugins,
|
||||
and cover use cases for overriding plugins at the end.
|
||||
|
||||
Depending on the evaluation setup, some (or all) of these plugins will be used
|
||||
during evaluation. For example, during an exclusively client-side evaluation,
|
||||
only the client plugin is needed.
|
||||
|
||||
Client Plugins
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
During exclusively client-side evolution, the evaluator will start the engine
|
||||
with the strategy under evaluation, and then run the client plugin. During
|
||||
server-side evolution, the evaluator will run the engine on the server-side,
|
||||
start the server plugin, and then start the client plugin via an SSH session to
|
||||
the remote client worker. (See :ref:`Adding a Worker` on how external workers
|
||||
can be used).
|
||||
|
||||
The client plugin subclasses from the PluginClient object. To tour through the
|
||||
API, we will walk through the development of a custom client plugin.
|
||||
|
||||
Writing Our Own
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Let us write a fitness function to test Iran's whitelisting system.
|
||||
|
||||
Iran's protocol whitelister was a recently deployed new censorship mechanism to
|
||||
censor non-whitelisted protocols on certain ports (53, 80, 443). We deployed
|
||||
Geneva against the whitelister, and discovered multiple ways to evade it in just
|
||||
one evolution of the genetic algorithm. (The results of that investigation is
|
||||
located `here <https://geneva.cs.umd.edu/posts/iran-whitelister>`_).
|
||||
|
||||
The whitelister worked by checking the first 2 packets of a flow, and if they
|
||||
did not match a fingerprint, it would destroy the flow.
|
||||
|
||||
In order to run Geneva against whitelister, we will define a client plugin that
|
||||
will try to trigger the whitelister and record whether the whitelister
|
||||
successfully censored its connection or not.
|
||||
|
||||
First, let's make a new folder in the :code:`plugins/` directory called
|
||||
"whitelister". We'll create a "client.py" and create the plugin object as
|
||||
a subclass of ClientPlugin.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class WhitelisterClient(ClientPlugin):
|
||||
"""
|
||||
Defines the whitelister client.
|
||||
"""
|
||||
name = "whitelister"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the whitelister client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
|
||||
Done! Next, let's define argument parsing for this plugin. Geneva uses a
|
||||
pass-through system of argument parsing: when command-line arguments are
|
||||
specified, evolve.py parses the options it knows and passes the rest to the
|
||||
evaluator. The evaluator prases the options it knows, and passes the list to the
|
||||
plugins. This allows developers to easily add their own arguments just to their
|
||||
plugin and use them from the command-line without changing any of the
|
||||
intermediate code.
|
||||
|
||||
In this case, we need our client to make a TCP connection to a server located
|
||||
outside of Iran to send our whitelister triggering messages to. Let's add an
|
||||
argument so the user can specify which server to connect to.
|
||||
|
||||
We can do this by adding a :code:`get_args` static method. The evaluator will
|
||||
call this method when the plugin is created and give it the full command line
|
||||
list, so the plugin is free to parse it how it chooses. For this example, we
|
||||
will use the standard :code:`argparse` library.
|
||||
|
||||
Since the superclass also defines args, we'll pass the command line list up to
|
||||
the super class as well to collect those arguments.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='Whitelister Client')
|
||||
|
||||
parser.add_argument('--server', action='store', help="server to connect to")
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
Now, we just need to define a :code:`run()` method. The :code:`run()` method is
|
||||
called by the evaluator to run the plugin. It provides the parsed arguments, a
|
||||
logger to log with, and a reference to an instance of the strategy engine that
|
||||
is running the strategy (see :ref:`Engine` for more information on how the
|
||||
engine works.)
|
||||
|
||||
Let's start by defining the run method. We'll pull out the argument for the
|
||||
server we defined earlier, connect to it with a python socket, and then just
|
||||
send "G", "E", and "T" in separate messages to trigger the whitelister. Since
|
||||
the whitelister censors connections by blackholing them, if the strategy failed
|
||||
to defeat the whitelister, we would expect our network connection to timeout;
|
||||
if we can send our messages and get a response from the server, the strategy
|
||||
under evaluation may have defeated the whitelister.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to open a socket, send two messages, and see if the messages
|
||||
time out.
|
||||
"""
|
||||
fitness = 0
|
||||
port = int(args["port"])
|
||||
server = args["server"]
|
||||
try:
|
||||
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
client.settimeout(3)
|
||||
client.connect((server, port))
|
||||
client.sendall(b"G")
|
||||
time.sleep(0.25)
|
||||
client.sendall(b"E")
|
||||
time.sleep(0.25)
|
||||
client.sendall(b"T\r\n\r\n")
|
||||
server_data = client.recv(1024)
|
||||
logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore'))
|
||||
if server_data:
|
||||
fitness += 100
|
||||
else:
|
||||
fitness -= 90
|
||||
client.close()
|
||||
# ...
|
||||
|
||||
Now we just need to define the error handling for this code. This is critical to
|
||||
the fitness function: we want to kill off strategies that damage the underlying
|
||||
TCP connection, so Geneva does not waste time searching this space of
|
||||
strategies.
|
||||
|
||||
Our goal is to set the fitness metric such that a *censorship event has a higher
|
||||
fitness than the strategy damaging the connection*. Since we can distinguish
|
||||
these cases based on the socket error, we will set a lower fitness if any other
|
||||
exception is raised besides the timeout.
|
||||
|
||||
Lastly, we'll inflate the numerical fitness metric to make it a larger number.
|
||||
The evaluator does additional punishments to the fitness score based on the
|
||||
strategy (see :ref:`Strategy Evaluation`), so we want the number to be
|
||||
sufficiently large to not push succeeding strategies to negative numbers.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
except socket.timeout:
|
||||
logger.debug("Client: Timeout")
|
||||
fitness -= 90
|
||||
except socket.error as exc:
|
||||
fitness -= 100
|
||||
logger.exception("Socket error caught in client echo test.")
|
||||
finally:
|
||||
logger.debug("Client finished whitelister test.")
|
||||
return fitness * 4
|
||||
|
||||
|
||||
Putting it all together:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class WhitelisterClient(ClientPlugin):
|
||||
"""
|
||||
Defines the whitelister client.
|
||||
"""
|
||||
name = "whitelister"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the whitelister client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='Whitelister Client')
|
||||
|
||||
parser.add_argument('--server', action='store', help="server to connect to")
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to open a socket, send two messages, and see if the messages
|
||||
time out.
|
||||
"""
|
||||
fitness = 0
|
||||
port = int(args["port"])
|
||||
server = args["server"]
|
||||
try:
|
||||
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
client.settimeout(3)
|
||||
client.connect((server, port))
|
||||
client.sendall(b"G")
|
||||
time.sleep(0.25)
|
||||
client.sendall(b"E")
|
||||
time.sleep(0.25)
|
||||
client.sendall(b"T\r\n\r\n")
|
||||
server_data = client.recv(1024)
|
||||
logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore'))
|
||||
if server_data:
|
||||
fitness += 100
|
||||
else:
|
||||
fitness -= 90
|
||||
client.close()
|
||||
except socket.timeout:
|
||||
logger.debug("Client: Timeout")
|
||||
fitness -= 90
|
||||
except socket.error as exc:
|
||||
fitness -= 100
|
||||
logger.exception("Socket error caught in client echo test.")
|
||||
finally:
|
||||
logger.debug("Client finished whitelister test.")
|
||||
return fitness * 4
|
||||
|
||||
Server Plugins
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
Coming soon!
|
||||
|
||||
Override Plugins
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Coming soon!
|
|
@ -0,0 +1,42 @@
|
|||
Adding a Worker
|
||||
===============
|
||||
|
||||
Below is how to add a new external client worker to Geneva. An external client worker is an external, SSH-accessible machine under the control of the user running Geneva for the purpose of performing strategy evaluation from outside the censored regime.
|
||||
|
||||
Geneva expects each of its worker to be defined in the :code:`workers` folder.
|
||||
|
||||
For this section, let us assume we are trying to allow Geneva to use a new external worker located in China.
|
||||
|
||||
First, make a new subfolder for that worker under the :code:`workers/` directory.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# mkdir workers/test
|
||||
# ls workers/
|
||||
example test
|
||||
|
||||
Each worker is defined by a :code:`worker.json` file located inside its subfolder.
|
||||
|
||||
The structure of the worker looks like this:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"name": "test",
|
||||
"ip": "<ip_here>",
|
||||
"hostname": "<hostname>",
|
||||
"username": "user",
|
||||
"password": null,
|
||||
"port": 22,
|
||||
"python": "python3",
|
||||
"city": "Bejing",
|
||||
"keyfile": "example.pem",
|
||||
"country": "China",
|
||||
"geneva_path": "~/geneva"
|
||||
}
|
||||
|
||||
If passwordless SSH is used, you can optionally specify a keyfile for it to SSH with.
|
||||
|
||||
Once this is defined, we can specify :code:`--external-client test` during strategy evaluation, and the evaluator will SSH to this worker for training!
|
||||
|
||||
.. note:: Remember, external client workers must have Geneva cloned to the directory specified in :code:`geneva_path` and depencies set up before use.
|
|
@ -0,0 +1,37 @@
|
|||
Engine
|
||||
======
|
||||
|
||||
The strategy engine (:code:`engine.py`) applies a strategy to a network connection. The engine works by capturing all traffic to/from a specified port. Packets that match an active trigger are run through the associated action-tree, and packets that emerge from the tree are sent on the wire.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# python3 engine.py --server-port 80 --strategy "\/" --log debug
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] Engine created with strategy \/ (ID bm3kdw3r) to port 80
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] Configuring iptables rules
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p tcp --sport 80 -j NFQUEUE --queue-num 1
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p tcp --dport 80 -j NFQUEUE --queue-num 2
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p udp --sport 80 -j NFQUEUE --queue-num 1
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p udp --dport 80 -j NFQUEUE --queue-num 2
|
||||
|
||||
|
||||
The engine also has a Python API for using it in your application. It can be used as a context manager or invoked in the background as a thread.
|
||||
For example, consider the following simple application.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import os
|
||||
import engine
|
||||
|
||||
# Port to run the engine on
|
||||
port = 80
|
||||
# Strategy to use
|
||||
strategy = "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/"
|
||||
|
||||
# Create the engine in debug mode
|
||||
with engine.Engine(port, strategy, log_level="debug") as eng:
|
||||
os.system("curl http://example.com?q=ultrasurf")
|
||||
|
||||
This script creates an instance of the engine with a specified strategy, and that strategy will be running for everything within the context manager. When the context manager exits, the engine will clean itself up. See the :code:`examples/` folder for more use cases of the engine.
|
||||
|
||||
.. note:: Due to limitations of scapy and NFQueue, the engine cannot be used to communicate with localhost.
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
Strategy Evaluation
|
||||
===================
|
||||
|
||||
Strategy evaluation tries to answer the question, *"which censorship evasion
|
||||
strategies should survive and propagate in the next generation?"* This is the
|
||||
job of the evaluator (:code:`evaluator.py`). The :code:`evaluator` assigns a numerical
|
||||
fitness score to each strategy based on a fitness function. The actual numerical
|
||||
fitness score itself is unimportant: as long as a 'better' strategy has a higher
|
||||
fitness score than a 'worse' strategy, our fitness function will be sound.
|
||||
Specific fitness numbers will be used in this section as examples, but all that
|
||||
matters is the _comparison_ between fitness scores.
|
||||
|
||||
Since the goal of Geneva is to find a strategy that evades censorship, to test a
|
||||
strategy, we can simply send a forbidden query through a censor with a strategy
|
||||
running and see if the request gets censored. Geneva operates at the network
|
||||
(TCP/IP) layer, so it is completely application agnostic: whether :code:`curl` or
|
||||
Google Chrome is generating network traffic, the engine can capture and modify
|
||||
it.
|
||||
|
||||
Environment IDs
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
During each evaluation, every strategy under test is given a random identifier, called an 'environment id'. This is used to track each strategy during evaluation. As each strategy is evaluated, a log file is written out named after the environment ID. See :ref:`Logging` for more information on how logs are stored.
|
||||
|
||||
|
||||
Plugins
|
||||
^^^^^^^^
|
||||
|
||||
To allow for evolution for many different applications, Geneva has a system of
|
||||
plugins (in `plugins/`). A plugin is used to drive an application to make a
|
||||
forbidden request, and defines the fitness function for that application. These
|
||||
plugins provide evaluator with a common interface to launch them and get the
|
||||
fitness score back.
|
||||
|
||||
Geneva provides some plugins with fitness functions for common protocols out of
|
||||
the box. Whenever the evaluator or :code:`evolve.py` is run, you must specify
|
||||
which plugin it should use with :code:`--test-type <plugin>`, such as
|
||||
:code:`--test-type http`.
|
||||
|
||||
Plugins can define a client and optionally a server. The client will attempt to
|
||||
make a forbidden connection through the censor to an external server, or
|
||||
optionally if run from another computer, to an instance of the server plugin.
|
||||
|
||||
To evaluate a strategy, the evaluator will initialize the engine with the strategy,
|
||||
launch the plugin client, and record the fitness returned by the plugin.
|
||||
|
||||
See :ref:`Adding New Plugins` for more details on how plugins work, how they can override behavior in the evaluator, and how to add new ones to Geneva.
|
||||
|
||||
Fitness Functions
|
||||
=================
|
||||
|
||||
Unlike many other machine learning scenarios, we have no gradient to learn against; censors give us only a binary signal (censored or not) to learn from. Therefore, we can't just write a fitness function that will directly guide us to the answer. Instead, we will use the fitness function to encourage the genetic algorithm to search the space of strategies that keeps the TCP connection alive.
|
||||
|
||||
As mentioned above, this guide might have specific fitness #s in examples, but the actual numeric values are not important - what is important is that a "bad" strategy gets a lower fitness number than a "good" strategy. We will use the fitness function to define a hierarchy of strategies.
|
||||
|
||||
The comparison order used by Geneva, ordered from best to worst:
|
||||
- Strategy that does not get censored and generates a minimal number of packets, no unused triggers, and minimal size
|
||||
- Strategy that does not get censored, but has unused actions, is too large, or imparts overhead
|
||||
- Strategy that gets censored
|
||||
- Strategy that does not trigger on any packets but gets censored
|
||||
- Strategy that breaks the underlying TCP connection
|
||||
- Empty strategy
|
||||
|
||||
Accomplishing this hierarchy is relatively straightforward - when evaluating a strategy, we assign a numerical fitness score such that strategies can be compared to one another and will be sorted according to this list.
|
||||
|
||||
This is done to guide the genetic algorithm in its search through the space of strategies. For example, consider a population pool that has 4 empty strategies, and one strategy that breaks the TCP connection. After evaluation, the empty strategies would be killed off, and the strategy that runs is propagated. When the offspring of this strategy mutate, if one of them no longer breaks the underlying TCP connection, it will be considered more fit than the other strategies, and so on.
|
||||
|
||||
This hierarchy accomplishes a significant *search space reduction*. Instead of Geneva fuzzing the entire space of possible strategies (for which there are many!), it instead quickly eliminates strategies that break the underlying connection and encourages the genetic algorithm to concentrate effort on only those strategies that keep the underlying connection alive.
|
||||
|
||||
Said another way, this hierarchy allows Geneva to differentiate between a strategy shooting ourselves in the foot, and being caught by the censor.
|
|
@ -0,0 +1,268 @@
|
|||
Running the Evaluator
|
||||
=====================
|
||||
|
||||
Since Geneva only needs to run on one side of the connection, Geneva is flexible as to where and how it can be run. Strategy evaluation can be done either on the server-side or the client-side. When used on the server-side, it can drive an external client to itself (or another server) for testing, or even act as a NAT-ing middlebox to interpose between the external client and true server.
|
||||
|
||||
The evaluator also allows plugins to override its default logic for single strategy evaluation, or for evaluating the entire strategy pool. See :ref:`Adding New Plugins` for more detail on writing your own plugin.
|
||||
|
||||
|
||||
Client-side Evaluation
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The most common use case for Geneva is client-side evaluation. In this mode,
|
||||
Geneva is run at the client-side inside a censored regime, trying to make a
|
||||
query to a forbidden resource located outside the censored regime. For this
|
||||
example, we will use the HTTP plugin. The HTTP plugin creates a forbidden HTTP
|
||||
GET request (such as `example.com?q=ultrasurf` in China).
|
||||
|
||||
The evaluator will start the engine, launch the client http plugin (which will
|
||||
make a request), and then the client plugin will record if the request succeeded
|
||||
or not. Under the hood, the GET request generated by the client HTTP plugin is
|
||||
caught by the engine (which modifies the traffic according to the strategy it is
|
||||
running) before sending it.
|
||||
|
||||
This effectively looks like this:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
Censored Regime
|
||||
---------------------------------------------------------+
|
||||
|
|
||||
Client - Geneva |
|
||||
+---------------------------------------------+ |
|
||||
| evaluator | | Server
|
||||
| +-----------+ strategy | | +-----------------+
|
||||
| | http | GET /bad +----------+ | +----------+ | |
|
||||
| | plugin <--------------> engine <-----------> censor <--------> forbidden.org |
|
||||
| | | +----------+ | +----------+ | |
|
||||
| +-----------+ | | +-----------------+
|
||||
| | |
|
||||
+---------------------------------------------+ |
|
||||
|
|
||||
|
|
||||
|
|
||||
---------------------------------------------------------+
|
||||
|
||||
|
||||
.. note:: For the purpose of this guide, examples will be given using :code:`evolve.py`. :code:`evolve.py` has an option :code:`--eval-only` to run the evaluator on a strategy with given parameters and exit. As usual, :code:`--eval-only` can also be replaced with parameters to the genetic algorithm to start the full genetic algorithm, instead of a single strategy evaluation.
|
||||
|
||||
To accomplish this, we can simply run:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --eval-only "\/" --test-type http --server forbidden.org --log debug
|
||||
|
||||
This will start :code:`evolve.py`, which will launch the `evaluator` with the
|
||||
`http` plugin, configured to make a request to `forbidden.org`, in debug mode.
|
||||
|
||||
|
||||
Server-side Evaluation
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Beyond using Geneva from the client-side, we can also use it from the server-side.
|
||||
|
||||
In this mode, Geneva can learn strategies that work from the server-side, subverting censorship on client's behalf.
|
||||
This requires a copy of Geneva to be checked out on both sides of the connection. To do this, we must specify the :code:`--server-side` flag.
|
||||
|
||||
Geneva will first start up a server for the specified plugin, and start the engine with the given strategy in front of the server. It next SSHes into an external client worker located inside a censored regime and runs the specified plugin through the SSH session.
|
||||
That plugin will generate a forbidden request to our server, and the plugin on the external client records if the request succeeded or not; the evaluator retrieves this fitness over the SSH session and evaluation is complete. This looks like this:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
Censored Regime
|
||||
+------------------------------+
|
||||
| Server - Geneva
|
||||
| +---------------------------------+
|
||||
Client | XXXXXXXXXX | |
|
||||
+------------+ | XX XX | strategy +-----------+ |
|
||||
| | GET /bad +----------+ X X | +--------+ | | |
|
||||
| curl <-------------> censor <------> Internet <----------> engine <---> evaluator | |
|
||||
| | +----------+ X X | +--------+ | | |
|
||||
+-----^------+ | XX XX | +-----+-----+ |
|
||||
| | XXXXXXXXXX | | |
|
||||
| | +-----------------------+---------+
|
||||
| | |
|
||||
+-----+------------------------+ |
|
||||
| |
|
||||
| 1. SSHs into the client and drives the request |
|
||||
+----------------------------------------------------------------------------+
|
||||
|
||||
To run this with :code:`evolve.py` to evaluate an empty strategy (:code:`\/`), we can do:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --test-type http --public-ip <mypublicip> --external-client example
|
||||
--log debug --eval-only "\/" --server-side
|
||||
|
||||
Note that we specified two additional options here: the public IP address of the test computer (used for making the request), and the external client worker we want to use.
|
||||
|
||||
.. note:: See :ref:`Adding A Worker` on how to configure external workers for Geneva.
|
||||
|
||||
External Clients with Local Servers
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Alternatively, we can evaluate the strategies on the client side, but have Geneva host the server for us so we can observe the traffic on both sides of the connection. By simply not specifying the :code:`--server-side` flag, we can accomplish this.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
Censored Regime
|
||||
+------------------------------------------+
|
||||
| Server + Geneva
|
||||
| +-------------------+
|
||||
External Client | | |
|
||||
+------------+ strategy | | +-----------+ |
|
||||
| | GET /bad +--------+ +----+-----+ | | | |
|
||||
| curl <------------> engine <----> censor <--------> evaluator | |
|
||||
| | +--------+ +----+-----+ | | | |
|
||||
+-----^------+ | | +-----+-----+ |
|
||||
| | | | |
|
||||
| | +-------------------+
|
||||
| | |
|
||||
+------------------------------------------+ |
|
||||
| |
|
||||
| 1. SSHs into the client and drives the request |
|
||||
+--------------------------------------------------------+
|
||||
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --test-type http --public-ip <mypublicip> --external-client example
|
||||
--log debug --eval-only "\/" --server-side
|
||||
|
||||
|
||||
|
||||
External Clients with External Servers
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Alternatively, the client can be driven to a different server from outside the censored regime with the :code:`'--external-server'` flag, and the server can be specified with :code:`'--server'`:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --test-type http --public-ip <mypublicip> --external-client example
|
||||
--log debug --eval-only "\/" --external-server --server http://wikipedia.org
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
Geneva
|
||||
+--------------------------+
|
||||
| |
|
||||
| +-----------+ |
|
||||
| | evaluator | |
|
||||
| +-----+-----+ |
|
||||
| | |
|
||||
+-------------+------------+
|
||||
|
|
||||
SSHes & drives request |
|
||||
+----------------+
|
||||
|
|
||||
| Censored Regime
|
||||
+---------+----------------------------------------------+
|
||||
| |
|
||||
| External Client |
|
||||
+---------+-----------------------------------+ |
|
||||
| | | | Server
|
||||
| +-----v-----+ strategy | | +-----------------+
|
||||
| | http | GET /bad +----------+ | +-----+----+ | |
|
||||
| | plugin <--------------> engine <-----------> censor <--------> forbidden.org |
|
||||
| | | +----------+ | +-----+----+ | |
|
||||
| +-----------+ | | +-----------------+
|
||||
| | |
|
||||
+---------------------------------------------+ |
|
||||
|
|
||||
|
|
||||
|
|
||||
+--------------------------------------------------------+
|
||||
|
||||
|
||||
Engine as a Middlebox
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
There are many cases in which we cannot trigger a censor while communicating with servers we control. In these cases,
|
||||
we can use Geneva as a middlebox, and interpose between the client and server and apply the strategy in between.
|
||||
|
||||
To accomplish this, we will specify :code:`--act-as-middlebox`, and specify three additional routing options:
|
||||
|
||||
Routing options:
|
||||
- :code:`--routing-ip`: Internal IP address of the middlebox
|
||||
- :code:`--forward-ip`: IP address we are forwarding to
|
||||
- :code:`--sender-ip`: IP address we are forwarding from
|
||||
|
||||
.. note:: Because Geneva operates at the packet-level, it cannot be used out of the box as a general purpose NAT, as it does not do connection tracking.
|
||||
|
||||
In this mode, the external client will communicate directly with our IP address, which will in turn forward the packets to the specified destination after they are modified according to the strategy under evaluation.
|
||||
|
||||
To run the strategy engine as a middlebox and drive an external client, we can add additional routing options:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --test-type http --public-ip <mypublicip> --external-client example --log
|
||||
debug --eval-only "\/" --server-side --act-as-middlebox --routing-ip <myinternalip>
|
||||
--forward-ip <iptoforwardto> --sender-ip <ipofexternalclient>
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
1. SSHs into the client and drives the request
|
||||
+------------------------------------------------+
|
||||
| |
|
||||
| +---------------------+
|
||||
| Censored Regime | | |
|
||||
+----------------------------+ | +-----+-----+ |
|
||||
| | | | evaluator | |
|
||||
v | | +-----------+ |
|
||||
External Client | | | Server
|
||||
+------------+ | | strategy | +---------+
|
||||
| | GET /bad +----+-----+ | +--------+ | | |
|
||||
| curl <------------> censor <------------> engine <------------> bad.com |
|
||||
| | +----+-----+ | +--------+ | | |
|
||||
+------------+ | | | +---------+
|
||||
| +---------------------+
|
||||
|
|
||||
+----------------------------+
|
||||
|
||||
|
||||
.. note:: The :code:`--routing-ip` is NOT the public IP address of your machine, its the internal address. This means that if you're running on an EC2 machine, the :code:`--public-ip` is your external IP, but the :code:`--routing-ip` is the IP you see when you run :code:`'ifconfig'`.
|
||||
|
||||
Internal Evaluation with Docker
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
For the purpose of internal testing and fitness function development, Geneva provides a set of simple mock censors (see :code:`censors/`). The evaluator can be configured to train against these censors.
|
||||
|
||||
.. note:: In order to do internal evaluation against these mock censors, you must have set up Geneva's base Docker container. See :ref:`Setup` for how to do this.
|
||||
|
||||
When used with Docker, Geneva will spin up three docker containers: a client, a
|
||||
censor, and a server, and configure the networking routes such that the client
|
||||
and server communicate through the censor. To evaluate strategies, Geneva will
|
||||
run the plugin client inside the client and attempt to communicate with the
|
||||
server through the censor.
|
||||
|
||||
Each docker container used by the evaluator runs out of the same base container.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
evaluator
|
||||
+--------------------------------------------------------------+
|
||||
| |
|
||||
| Client Censor Server |
|
||||
| +----------+ +----------+ +----------+ |
|
||||
| | | | | | | |
|
||||
| | client | | censor | | server | |
|
||||
| | <-----+-----> <-----+-----> | |
|
||||
| | plugin | ^ | | ^ | plugin | |
|
||||
| | | | | | | | | |
|
||||
| +----------+ | +----------+ | +----------+ |
|
||||
| container | container | container |
|
||||
+--------------------------------------------------------------+
|
||||
| |
|
||||
| |
|
||||
+------------------+
|
||||
engine can be run on either side
|
||||
|
||||
|
||||
We can accomplish this by specifying the :code:`--censor <censor>` option. This will enable running with Docker, start the censor, and perform evaluation.
|
||||
Depending on whether :code:`--server-side` is specified, the engine will be run on either the client or server side.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --eval-only "" --test-type echo --censor censor2 --log debug
|
||||
|
||||
.. note:: :code:`--censor` cannot be used with :code:`--external-client` or :code:`--external-server`.
|
|
@ -0,0 +1,42 @@
|
|||
Evolution
|
||||
==========
|
||||
|
||||
Now that we have a concrete definition for a censorship evasion strategies and a way to use them, we can begin evolving new strategies with Geneva's genetic algorithm.
|
||||
|
||||
A genetic algorithm is a type of machine learning inspired by natural selection. Over the course of many *generations*, it optimizes a numerical *fitness* score of individuals; within each generation, individuals that have a low fitness will not survive to the next generation, and those with a high fitness score will survive and propagate.
|
||||
|
||||
Each individual in Geneva is a censorship evasion strategy. :code:`evolve.py` is the main driver for Geneva's genetic algorithm and maintains the population pool of these strategies.
|
||||
|
||||
Each generation is comprised of the following:
|
||||
1. Mutation/Crossover - randomly mutating/mating strategies in the pool
|
||||
2. Strategy Evaluation - assigning a numerical fitness score to each strategy - decides which strategies should live on to the next generation
|
||||
3. Selection - the selection process of which strategies should survive to the next generation
|
||||
|
||||
For more detail on mutation/crossover or the selection process, see `our papers <https://geneva.cs.umd.edu/papers>`_ or the code in :code:`evolve.py` for more detail. :code:`evolve.py` exposes options to control hyperparameters of mutation/crossover, as well as other advanced options for rejecting mutations it has seen before.
|
||||
|
||||
Strategy evaluation is significantly more complex, and will be covered in depth in the next section.
|
||||
|
||||
For most of this documentation, we will show examples of using :code:`evolve.py` with the :code:`--eval-only <strategy_here>` option. This instructs :code:`evolve.py` not to start the full genetic algorithm, but to instead perform a single strategy evaluation with the given parameters. :code:`--eval-only` can be replaced with parameters to the genetic algorithm to start strategy evolution.
|
||||
|
||||
|
||||
Argument Parsing
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Geneva uses a pass-through system of argument parsing. Different parts of the system define which arguments they care about, and they will parse just those args out.
|
||||
If :code:`--help` is used, :code:`evolve.py` will collect the help messages for the relevant components (evaluator, plugins, etc).
|
||||
|
||||
|
||||
Population Control
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The two most important command line options for controlling evolutions are :code:`--population` and
|
||||
:code:`--generations`. These control the population size and number of generations evolution will
|
||||
take place for, respectively.
|
||||
|
||||
Geneva will not automatically halt evolution after population convergence occurs.
|
||||
|
||||
|
||||
Seeding Population
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Geneva allows you to seed the starting population pool of strategies using the :code:`--seed <strategy>` parameter. This will create the initial population pool with nothing but copies of the given strategy. Mutation is applied to each individual before evaluation begins, so the first generation is not solely one individual being evaluated over and over again.
|
|
@ -0,0 +1,86 @@
|
|||
How it Works
|
||||
==============
|
||||
|
||||
See our `website <https://censorship.ai>`_ or our `research papers <https://geneva.cs.umd.edu/papers>`_ for an in-depth read on how Geneva works.
|
||||
|
||||
This documentation will provide a walkthrough of the main concepts behind Geneva, the main components of the codebase, and how they can be used.
|
||||
|
||||
Censorship Evasion Strategies
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
A censorship evasion strategy is simply a *description of how network traffic should be modified*. A strategy is **not
|
||||
code**, it is a description that tells Geneva's strategy engine how it should manipulate network traffic.
|
||||
|
||||
The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted.
|
||||
|
||||
A censorship evasion strategy composed of one or more packet-level building blocks. Geneva's core building blocks are:
|
||||
|
||||
1. :code:`duplicate`: takes one packet and returns two copies of the packet
|
||||
2. :code:`drop`: takes one packet and returns no packets (drops the packet)
|
||||
3. :code:`tamper`: takes one packet and returns the modified packet
|
||||
4. :code:`fragment`: takes one packet and returns two fragments or two segments
|
||||
|
||||
Since :code:`duplicate` and :code:`fragment` introduce *branching*, these actions are composed into a binary-tree structure called an *action tree*.
|
||||
|
||||
Each tree also has a *trigger*. The trigger describes which packets the tree should run on, and the tree describes what should happen to each of those packets when the trigger fires. Once a trigger fires on a packet, it pulls the packet into the tree for modifications, and the packets that emerge from the tree are sent on the wire. Recall that Geneva operates at the packet level, therefore all triggers are packet-level triggers.
|
||||
|
||||
Multiple action trees together form a *forest*. Geneva handles outbound and inbound packets differently, so strategies are composed of two forests: an outbound forest and an inbound forest.
|
||||
|
||||
Consider the following example of a simple Geneva strategy.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
+---------------+ triggers on TCP packets with the flags
|
||||
| TCP:flags:A | <-- field set to 'ACK' - matching packets
|
||||
+-------+-------+ are captured and pulled into the tree
|
||||
|
|
||||
+---------v---------+ makes two copies of the given packet.
|
||||
duplicate <-- the tree is processed with an inorder
|
||||
+---------+---------+ traversal, so the left side is run first
|
||||
|
|
||||
+-------------+------------+
|
||||
| |
|
||||
+------------v----------+ v <-- dupilcate has no right child
|
||||
tamper
|
||||
{TCP:flags:replace:R} <-- parameters to this action describe how
|
||||
+------------+----------+ the packet should be tampered
|
||||
|
|
||||
+------------v----------+
|
||||
tamper
|
||||
{TCP:chksum:corrupt}
|
||||
+------------+----------+
|
||||
|
|
||||
v <-- packets that emerge from an in-order traversal
|
||||
of the leaves are sent on the wire
|
||||
|
||||
Strategy DNA Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
These strategies can be arbitrarily complicated, and Geneva defines a well-formatted string syntax for
|
||||
unambiguously expressing strategies.
|
||||
|
||||
A strategy divides how it handles outbound and inbound packets: these are separated in the DNA by a
|
||||
"\\/". Specifically, the strategy format is :code:`<outbound forest> \/ <inbound forest>`. If :code:`\/` is not
|
||||
present in a strategy, all of the action trees are in the outbound forest.
|
||||
|
||||
Both forests are composed of action trees, and each forest is allowed an arbitrarily many trees.
|
||||
|
||||
Action trees always start with a trigger, which is formatted as: :code:`[<protocol>:<field>:<value>]`. For example, the trigger: :code:`[TCP:flags:S]` will run its corresponding tree whenever it sees a :code:`TCP` packet with the :code:`flags` field set to :code:`SYN`. If the corresponding action tree is :code:`[TCP:flags:S]-drop-|`, this action tree will cause the engine to drop any :code:`SYN` packets. :code:`[TCP:flags:S]-duplicate-|` will cause the engine to duplicate any SYN packets.
|
||||
|
||||
Triggers also can contain an optional 4th parameter for *gas*, which describes the number of times a trigger can fire. The triger :code:`[IP:version:4:4]` will run only on the first 4 IPv4 packets it sees. If the gas is negative, the trigger acts as a *bomb* trigger, which means the trigger will not fire until a certain number of applicable packets have been seen. For example, the trigger :code:`[IP:version:4:-2]` will trigger only after it has seen two matching packets (and it will not trigger on those first packets).
|
||||
|
||||
Syntactically, action trees end with :code:`-|`.
|
||||
|
||||
Depending on the type of action, some actions can have up to two children (such as :code:`duplicate`). These are represented
|
||||
with the following syntax: :code:`[TCP:flags:S]-duplicate(<left_child>,<right_child>)-|`, where
|
||||
:code:`<left_child>` and :code:`<right_child>` themselves are trees. If :code:`(,)` is not specified, any packets
|
||||
that emerge from the action will be sent on the wire. If an action only has one child (such as :code:`tamper`), it is always the left child. :code:`[TCP:flags:S]-tamper{<parameters>}(<left_child>,)-|`
|
||||
|
||||
Actions that have parameters specify those parameters within :code:`{}`. For example, giving parameters to the :code:`tamper` action could look like: :code:`[TCP:flags:S]-tamper{TCP:flags:replace:A}-|`. This strategy would trigger on TCP :code:`SYN` packets and replace the TCP :code:`flags` field to :code:`ACK`.
|
||||
|
||||
Putting this all together, below is the strategy DNA representation of the above diagram:
|
||||
:code:`[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/`
|
||||
|
||||
Geneva has code to parse this strategy DNA into strategies that can be applied to network traffic using the engine.
|
||||
|
||||
.. note:: Due to limitations of Scapy and NFQueue, actions that introduce branching (:code:`fragment`, :code:`duplicate`) are disabled for incoming action forests.
|
|
@ -0,0 +1,28 @@
|
|||
Logging
|
||||
=======
|
||||
|
||||
Geneva uses multiple loggers during execution. :code:`evolve.py` creates the parent logger, and creates a subfolder of the current time under the :code:`trials/` directory.
|
||||
|
||||
Within this directory, it creates 5 subfolders:
|
||||
- :code:`data`: used for misc. data related to strategy evaluation
|
||||
- :code:`flags`: used to write status files to set events
|
||||
- :code:`generations`: used to store the full generations and hall of fame after each generation
|
||||
- :code:`logs`: stores logs for evaluation
|
||||
- :code:`packets`: stores packet captures during strategy evolution
|
||||
|
||||
The two main log files used by :code:`evolve.py` are :code:`ga.log` and :code:`ga_debug.log` (everything in debug mode). As each strategy is evaluated, a :code:`<id>_engine.log`, :code:`<id>_server.log`, and :code:`<id>_client.log` files are generated.
|
||||
|
||||
For example, one run's output could be:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# ls trials
|
||||
2020-03-23_20:03:08
|
||||
|
||||
# ls trials/2020-03-23_20:03:08
|
||||
data flags generations logs packets
|
||||
|
||||
# ls trials/2020-03-23_20:03:08/logs
|
||||
ga.log ga_debug.log zhak1n81_client.log zhak1n81_engine.log zhak1n81_server.log
|
||||
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
Putting it all Together
|
||||
=======================
|
||||
|
||||
Now that we know how to leverage the framework, this short section will give a high level of
|
||||
how to put it all together and run Geneva's genetic algorithm yourself.
|
||||
|
||||
The two most important command line options for controlling evolutions are :code:`--population` and
|
||||
:code:`--generations`. These control the population size and number of generations evolution will
|
||||
take place for, respectively.
|
||||
|
||||
Generally, these need not be very large. Geneva intentionally does strategy evaluation serially & slowly, so increasing the size dramatically will make evolution take longer. For our existing research papers, the population count rarely exceeded 300.
|
||||
|
||||
For example, to run a client-side evolution against HTTP censorship with a population pool of 200 and 25 generations, the following can be used:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 evolve.py --population 200 --generations 25 --test-type http --server forbidden.org
|
||||
|
||||
Before running any evolution, it is recommended to spot test the plugins against whatever censor is used as the adversary to confirm the fitness function properly sets up the desirable hierarchy of individuals as specified in :ref:`Fitness Functions`: strategies that break the connection get lower fitness than those that get censored, which get a lower fitness than those that succeed.
|
||||
|
||||
And that's it!
|
|
@ -0,0 +1,13 @@
|
|||
Automated Tests
|
||||
===============
|
||||
|
||||
Geneva has a system of automated tests in the :code:`tests/` directory, powered by pytest. Unless you are doing
|
||||
modifications to the source code, you can generally ignore these.
|
||||
|
||||
If you need to run them yourself, you can do so with:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
# python3 -m pytest -sv tests/
|
||||
|
||||
To put the tests in debug mode, you can add :code:`--evolve-logger debug`.
|
|
@ -0,0 +1,45 @@
|
|||
.. geneva documentation master file, created by
|
||||
sphinx-quickstart on Fri Apr 10 12:35:13 2020.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to Geneva's documentation!
|
||||
==================================
|
||||
|
||||
|
||||
**Disclaimer:** Running Geneva or Geneva's strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it.
|
||||
|
||||
.. toctree::
|
||||
:caption: Getting Started:
|
||||
|
||||
intro/introduction
|
||||
intro/setup
|
||||
intro/gettingstarted
|
||||
|
||||
.. toctree::
|
||||
:caption: Usage:
|
||||
|
||||
howitworks/howitworks
|
||||
howitworks/engine
|
||||
howitworks/evolution
|
||||
howitworks/evaluation
|
||||
howitworks/evaluator
|
||||
howitworks/addingaworker
|
||||
howitworks/logging
|
||||
howitworks/testing
|
||||
howitworks/puttingittogether
|
||||
|
||||
.. toctree::
|
||||
:caption: Extending Geneva:
|
||||
|
||||
extending/plugins
|
||||
extending/actions
|
||||
extending/contributing
|
||||
|
||||
.. toctree::
|
||||
:glob:
|
||||
:caption: API Reference:
|
||||
|
||||
api/*
|
||||
api/actions/*
|
||||
api/plugins/*
|
|
@ -0,0 +1,93 @@
|
|||
Getting Started
|
||||
=================
|
||||
|
||||
See our `website <https://censorship.ai>`_ or our `research papers <https://geneva.cs.umd.edu/papers>`_ for an in-depth read on how Geneva works.
|
||||
|
||||
This documentation will provide a walkthrough of the main concepts behind Geneva, the main components of the codebase, and how they can be used.
|
||||
|
||||
This section will give a high level overview on how Geneva works; before using it, you are **strongly recommended** to read through :ref:`How it Works`.
|
||||
|
||||
|
||||
What is a Strategy?
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A censorship evasion strategy is simply a *description of how network traffic
|
||||
should be modified*. A strategy is **not code**, it is a description that tells the
|
||||
*strategy engine* how it should manipulate network traffic.
|
||||
|
||||
The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted.
|
||||
|
||||
Strategies & Species
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Because Geneva commonly identifies many different strategies, we have defined a
|
||||
*taxonomy* to classify strategies into.
|
||||
|
||||
The Strategy taxonomy is as follows, ordered from most general to most specific:
|
||||
- 1. Species: The overarching bug a strategy exploits
|
||||
- 2. Subspecies: The mechanism used to exploit the bug
|
||||
- 3. Variant: Salient wireline differences using the same bug mechanism
|
||||
|
||||
The highest level classification is *species*,
|
||||
a broad class of strategies classified by the type of weakness it exploits in a
|
||||
censor implementation. :code:`TCB Teardown` is an example of one such species; if the
|
||||
censor did not prematurely teardown TCBs, all the strategies in this species
|
||||
would cease to function.
|
||||
|
||||
Within each species, different *subspecies* represent
|
||||
unique ways to exploit the weakness that defines the strategy. For example,
|
||||
injecting an insertion TCP :code:`RST` packet would comprise one subspecies within the
|
||||
TCB Teardown species; injecting a TCP :code:`FIN` would comprise another.
|
||||
|
||||
Within each
|
||||
subspecies, we further record *variants*, unique strategies that leverage the same
|
||||
attack vector, but do so slightly differently: corrupting the checksum field on
|
||||
a :code:`RST` packet is one variant of the :code:`TCB Teardown w/ RST` subspecies of the :code:`TCB Teardown` species;
|
||||
corrupting the :code:`ack` field is another.
|
||||
|
||||
We refer to specific individuals as *extinct*
|
||||
if they once worked against a censor but are no longer effective (less than 5%
|
||||
success rate). That formerly successful approaches could, after a few years, become
|
||||
ineffective lends further motivation for a technique that can quickly learn new
|
||||
strategies.
|
||||
|
||||
|
||||
Running a Strategy
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
For a fuller description of the DNA syntax, see :ref:`Censorship Evasion Strategies`.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# python3 engine.py --server-port 80 --strategy "\/" --log debug
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] Engine created with strategy \/ (ID bm3kdw3r) to port 80
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] Configuring iptables rules
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p tcp --sport 80 -j NFQUEUE --queue-num 1
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p tcp --dport 80 -j NFQUEUE --queue-num 2
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p udp --sport 80 -j NFQUEUE --queue-num 1
|
||||
2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p udp --dport 80 -j NFQUEUE --queue-num 2
|
||||
|
||||
|
||||
Note that if you have stale :code:`iptables` rules or other rules that rely on Geneva's default queues,
|
||||
this will fail. To fix this, remove those rules.
|
||||
|
||||
Strategy Library
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Geneva has found dozens of strategies that work against censors in China,
|
||||
Kazakhstan, India, and Iran. We include several of these strategies in
|
||||
`strategies.md <https://github.com/kkevsterrr/geneva>`_ . Note that this file contains success rates for
|
||||
each individual country; a strategy that works in one country may not work as
|
||||
well as other countries.
|
||||
|
||||
Researchers have observed that strategies may have differing success rates based
|
||||
on your exact location. Although we have not observed this from our vantage
|
||||
points, you may find that some strategies may work differently in a country we
|
||||
have tested. If this is the case, don't be alarmed. However, please feel free to
|
||||
reach out to a member of the team directly or open an issue on this page so we
|
||||
can track how the strategies work from other geographic locations.
|
||||
|
||||
Disclaimer
|
||||
==========
|
||||
|
||||
Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it.
|
|
@ -0,0 +1,28 @@
|
|||
Introduction
|
||||
===================================
|
||||
|
||||
Geneva is an artificial intelligence tool that defeats censorship by exploiting
|
||||
bugs in censors, such as those in China, India, and Kazakhstan. Unlike many
|
||||
other anti-censorship solutions which require assistance from outside the
|
||||
censoring regime (Tor, VPNs, etc.), Geneva runs strictly on one side of the
|
||||
connection (either the client or server side).
|
||||
|
||||
Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion
|
||||
strategies and has found several previously unknown bugs in censors. Geneva's
|
||||
strategies manipulate the network stream to confuse the censor without impacting
|
||||
the client/server communication. This makes Geneva effective against many types
|
||||
of in-network censorship (though it cannot be used against IP-blocking
|
||||
censorship).
|
||||
|
||||
Geneva is composed of two high level components: its *genetic algorithm* (which
|
||||
it uses to evolve new censorship evasion strategies) and its *strategy engine*
|
||||
(which is uses to run an individual censorship evasion strategy over a network
|
||||
connection).
|
||||
|
||||
Geneva's `Github page <https://github.com/kkevsterrr/geneva>`_ contains the
|
||||
Geneva's full implementation: its genetic algorithm, strategy engine, Python
|
||||
API, and a subset of published strategies. With these tools, users and
|
||||
researchers alike can evolve new strategies or leverage existing strategies to
|
||||
evade censorship. To learn more about how Geneva works, see :ref:`How it
|
||||
Works`.
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
Setup
|
||||
=====
|
||||
|
||||
Geneva has been developed and tested for Centos or Debian-based systems. Due to
|
||||
limitations of netfilter and raw sockets, Geneva does not work on OS X or
|
||||
Windows at this time and requires *python3.6*.
|
||||
|
||||
Install netfilterqueue dependencies:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# sudo apt-get install build-essential python-dev libnetfilter-queue-dev libffi-dev libssl-dev iptables python3-pip
|
||||
|
||||
|
||||
Install Python dependencies:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# python3 -m pip install -r requirements.txt
|
||||
|
||||
Docker (Optional)
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
Geneva has an internal system that can be used to test strategies using Docker.
|
||||
This is largely used for testing fitness functions with the mock censors
|
||||
provided - it is **not used for training against real censors**. Due to
|
||||
limitations of raw sockets inside docker containers in many builds of Docker,
|
||||
Geneva cannot be used inside a docker container to communicate with hosts
|
||||
outside of Docker's internal network.
|
||||
|
||||
When used with Docker, Geneva will spin up three docker containers: a client, a
|
||||
censor, and a server, and configure the networking routes such that the client
|
||||
and server communicate through the censor. To evaluate strategies (see much more detail in the evaluation section),
|
||||
Geneva will run the plugin client inside the client and
|
||||
attempt to communicate with the server through the censor.
|
||||
|
||||
Each docker container used by the evaluator runs out of the same base container.
|
||||
|
||||
Build the base container with:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker build -t base:latest -f docker/Dockerfile .
|
||||
|
||||
Optionally, to manually run/inspect the docker image to explore the image, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker run -it base
|
|
@ -0,0 +1,35 @@
|
|||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=.
|
||||
set BUILDDIR=_build
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.http://sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
|
@ -0,0 +1,8 @@
|
|||
scapy
|
||||
anytree
|
||||
psutil
|
||||
netifaces
|
||||
requests
|
||||
dnspython
|
||||
paramiko
|
||||
tld
|
197
engine.py
197
engine.py
|
@ -1,8 +1,8 @@
|
|||
"""
|
||||
Engine
|
||||
Geneva Strategy Engine
|
||||
|
||||
Given a strategy and a server port, the engine configures NFQueue
|
||||
so the strategy can run on the underlying connection.
|
||||
Given a strategy and a server port, the engine configures NFQueue to capture all traffic
|
||||
into and out of that port so the strategy can run over the connection.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
@ -14,7 +14,10 @@ import subprocess
|
|||
import threading
|
||||
import time
|
||||
|
||||
import netfilterqueue
|
||||
try:
|
||||
import netfilterqueue
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from scapy.layers.inet import IP
|
||||
from scapy.utils import wrpcap
|
||||
|
@ -22,7 +25,7 @@ from scapy.config import conf
|
|||
|
||||
socket.setdefaulttimeout(1)
|
||||
|
||||
import actions.packet
|
||||
import layers.packet
|
||||
import actions.strategy
|
||||
import actions.utils
|
||||
|
||||
|
@ -30,28 +33,78 @@ BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
|||
|
||||
|
||||
class Engine():
|
||||
def __init__(self, server_port, string_strategy, server_side=False, environment_id=None, output_directory="trials", log_level="info"):
|
||||
def __init__(self, server_port,
|
||||
string_strategy,
|
||||
environment_id=None,
|
||||
server_side=False,
|
||||
output_directory="trials",
|
||||
log_level="info",
|
||||
enabled=True,
|
||||
in_queue_num=None,
|
||||
out_queue_num=None,
|
||||
forwarder=None,
|
||||
save_seen_packets=True,
|
||||
demo_mode=False):
|
||||
"""
|
||||
Args:
|
||||
server_port (int): The port the engine will monitor
|
||||
string_strategy (str): String representation of strategy DNA to apply to the network
|
||||
environment_id (str, None): ID of the given strategy
|
||||
server_side (bool, False): Whether or not the engine is running on the server side of the connection
|
||||
output_directory (str, 'trials'): The path logs and packet captures should be written to
|
||||
enabled (bool, True): whether or not the engine should be started (used for conditional context managers)
|
||||
in_queue_num (int, None): override the netfilterqueue number used for inbound packets. Used for running multiple instances of the engine at the same time. Defaults to None.
|
||||
out_queue_num (int, None): override the netfilterqueue number used for outbound packets. Used for running multiple instances of the engine at the same time. Defaults to None.
|
||||
save_seen_packets (bool, True): whether or not the engine should record and save packets it sees while running. Defaults to True, but it is recommended this be disabled on higher throughput systems.
|
||||
demo_mode (bool, False): whether to replace IPs in log messages with random IPs to hide sensitive IP addresses.
|
||||
"""
|
||||
self.server_port = server_port
|
||||
# whether the engine is running on the server or client side.
|
||||
# this affects which direction each out/in tree is attached to the
|
||||
# source and destination port.
|
||||
self.server_side = server_side
|
||||
self.overhead = 0
|
||||
self.seen_packets = []
|
||||
# Set up the directory and ID for logging
|
||||
actions.utils.setup_dirs(output_directory)
|
||||
if not environment_id:
|
||||
environment_id = actions.utils.get_id()
|
||||
|
||||
self.environment_id = environment_id
|
||||
self.forwarder = forwarder
|
||||
self.save_seen_packets = save_seen_packets
|
||||
if forwarder:
|
||||
self.sender_ip = forwarder["sender_ip"]
|
||||
self.routing_ip = forwarder["routing_ip"]
|
||||
self.forward_ip = forwarder["forward_ip"]
|
||||
|
||||
# Set up the directory and ID for logging
|
||||
if not output_directory:
|
||||
self.output_directory = "trials"
|
||||
else:
|
||||
self.output_directory = output_directory
|
||||
actions.utils.setup_dirs(self.output_directory)
|
||||
if not environment_id:
|
||||
self.environment_id = actions.utils.get_id()
|
||||
|
||||
# Set up a logger
|
||||
self.logger = actions.utils.get_logger(BASEPATH,
|
||||
output_directory,
|
||||
self.output_directory,
|
||||
__name__,
|
||||
"engine",
|
||||
environment_id,
|
||||
log_level=log_level)
|
||||
self.output_directory = output_directory
|
||||
self.server_side = server_side
|
||||
self.environment_id,
|
||||
log_level=log_level,
|
||||
demo_mode=demo_mode)
|
||||
# Warn if these are not provided
|
||||
if not environment_id:
|
||||
self.logger.warning("No environment ID given, one has been generated (%s)", self.environment_id)
|
||||
if not output_directory:
|
||||
self.logger.warning("No output directory specified, using the default (%s)" % self.output_directory)
|
||||
|
||||
# Used for conditional context manager usage
|
||||
self.enabled = enabled
|
||||
|
||||
# Parse the given strategy
|
||||
self.strategy = actions.utils.parse(string_strategy, self.logger)
|
||||
|
||||
# Setup variables used by the NFQueue system
|
||||
self.in_queue_num = in_queue_num or 1
|
||||
self.out_queue_num = out_queue_num or self.in_queue_num + 1
|
||||
self.out_nfqueue_started = False
|
||||
self.in_nfqueue_started = False
|
||||
self.running_nfqueue = False
|
||||
|
@ -71,22 +124,51 @@ class Engine():
|
|||
def __enter__(self):
|
||||
"""
|
||||
Allows the engine to be used as a context manager; simply launches the
|
||||
engine.
|
||||
engine if enabled.
|
||||
"""
|
||||
if self.enabled:
|
||||
self.initialize_nfqueue()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, tb):
|
||||
"""
|
||||
Allows the engine to be used as a context manager; simply stops the engine
|
||||
if enabled.
|
||||
"""
|
||||
if self.enabled:
|
||||
self.shutdown_nfqueue()
|
||||
|
||||
def do_nat(self, packet):
|
||||
"""
|
||||
NATs packet: changes the sources and destination IP if it matches the
|
||||
configured route, and clears the checksums for recalculating
|
||||
|
||||
Args:
|
||||
packet (layers.packet.Packet): packet to modify before sending
|
||||
|
||||
Returns:
|
||||
layers.packet.Packet: the modified packet
|
||||
"""
|
||||
if packet["IP"].src == self.sender_ip:
|
||||
packet["IP"].dst = self.forward_ip
|
||||
packet["IP"].src = self.routing_ip
|
||||
del packet["TCP"].chksum
|
||||
del packet["IP"].chksum
|
||||
elif packet["IP"].src == self.forward_ip:
|
||||
packet["IP"].dst = self.sender_ip
|
||||
packet["IP"].src = self.routing_ip
|
||||
del packet["TCP"].chksum
|
||||
del packet["IP"].chksum
|
||||
return packet
|
||||
|
||||
def mysend(self, packet):
|
||||
"""
|
||||
Helper scapy sending method. Expects a Geneva Packet input.
|
||||
"""
|
||||
try:
|
||||
if self.forwarder:
|
||||
self.logger.debug("NAT-ing packet.")
|
||||
packet = self.do_nat(packet)
|
||||
self.logger.debug("Sending packet %s", str(packet))
|
||||
self.socket.send(packet.packet)
|
||||
except Exception:
|
||||
|
@ -113,7 +195,8 @@ class Engine():
|
|||
self.in_nfqueue_started = True
|
||||
|
||||
nfqueue.run_socket(nfqueue_socket)
|
||||
except socket.timeout:
|
||||
# run_socket can raise an OSError on shutdown for some builds of netfilterqueue
|
||||
except (socket.timeout, OSError):
|
||||
pass
|
||||
except Exception:
|
||||
self.logger.exception("Exception out of run_nfqueue() (direction=%s)", direction)
|
||||
|
@ -124,9 +207,10 @@ class Engine():
|
|||
"""
|
||||
self.logger.debug("Configuring iptables rules")
|
||||
|
||||
port1, port2 = "dport", "sport"
|
||||
if self.server_side:
|
||||
# Switch source and destination ports if this evaluator is to run from the server side
|
||||
port1, port2 = "sport", "dport"
|
||||
if not self.server_side:
|
||||
port1, port2 = "dport", "sport"
|
||||
|
||||
out_chain = "OUTPUT"
|
||||
in_chain = "INPUT"
|
||||
|
@ -137,16 +221,23 @@ class Engine():
|
|||
add_or_remove = "D"
|
||||
cmds = []
|
||||
for proto in ["tcp", "udp"]:
|
||||
cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 1" %
|
||||
(add_or_remove, out_chain, proto, port1, self.server_port),
|
||||
"iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 2" %
|
||||
(add_or_remove, in_chain, proto, port2, self.server_port)]
|
||||
cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" %
|
||||
(add_or_remove, out_chain, proto, port1, self.server_port, self.out_queue_num),
|
||||
"iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" %
|
||||
(add_or_remove, in_chain, proto, port2, self.server_port, self.in_queue_num)]
|
||||
# If this machine is acting as a middlebox, we need to add the same rules again
|
||||
# in the opposite direction so that we can pass packets back and forth
|
||||
if self.forwarder:
|
||||
cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" %
|
||||
(add_or_remove, out_chain, proto, port2, self.server_port, self.out_queue_num),
|
||||
"iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" %
|
||||
(add_or_remove, in_chain, proto, port1, self.server_port, self.in_queue_num)]
|
||||
|
||||
for cmd in cmds:
|
||||
self.logger.debug(cmd)
|
||||
# If we're logging at DEBUG mode, keep stderr/stdout piped to us
|
||||
# If we're logging at debug mode, keep stderr/stdout piped to us
|
||||
# Otherwise, pipe them both to DEVNULL
|
||||
if actions.utils.get_console_log_level() == logging.DEBUG:
|
||||
if actions.utils.get_console_log_level() == "debug":
|
||||
subprocess.check_call(cmd.split(), timeout=60)
|
||||
else:
|
||||
subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60)
|
||||
|
@ -167,8 +258,8 @@ class Engine():
|
|||
self.out_nfqueue = netfilterqueue.NetfilterQueue()
|
||||
self.in_nfqueue = netfilterqueue.NetfilterQueue()
|
||||
# Bind them
|
||||
self.out_nfqueue.bind(1, self.out_callback)
|
||||
self.in_nfqueue.bind(2, self.in_callback)
|
||||
self.out_nfqueue.bind(self.out_queue_num, self.out_callback)
|
||||
self.in_nfqueue.bind(self.in_queue_num, self.in_callback)
|
||||
# Create our nfqueue sockets to allow for non-blocking usage
|
||||
self.out_nfqueue_socket = socket.fromfd(self.out_nfqueue.get_fd(),
|
||||
socket.AF_UNIX,
|
||||
|
@ -211,6 +302,9 @@ class Engine():
|
|||
if self.out_nfqueue:
|
||||
self.out_nfqueue.unbind()
|
||||
self.configure_iptables(remove=True)
|
||||
self.socket.close()
|
||||
self.out_nfqueue_socket.close()
|
||||
self.in_nfqueue_socket.close()
|
||||
|
||||
packets_path = os.path.join(BASEPATH,
|
||||
self.output_directory,
|
||||
|
@ -218,6 +312,7 @@ class Engine():
|
|||
"original_%s.pcap" % self.environment_id)
|
||||
|
||||
# Write to disk the original packets we captured
|
||||
if self.save_seen_packets:
|
||||
wrpcap(packets_path, [p.packet for p in self.seen_packets])
|
||||
|
||||
# If the engine exits before it initializes for any reason, these threads may not be set
|
||||
|
@ -237,10 +332,11 @@ class Engine():
|
|||
if not self.running_nfqueue:
|
||||
return
|
||||
|
||||
packet = actions.packet.Packet(IP(nfpacket.get_payload()))
|
||||
packet = layers.packet.Packet(IP(nfpacket.get_payload()))
|
||||
self.logger.debug("Received outbound packet %s", str(packet))
|
||||
|
||||
# Record this packet for a .pacp later
|
||||
if self.save_seen_packets:
|
||||
self.seen_packets.append(packet)
|
||||
|
||||
# Drop the packet in NFQueue so the strategy can handle it
|
||||
|
@ -253,6 +349,8 @@ class Engine():
|
|||
Handles processing an outbound packet through the engine.
|
||||
"""
|
||||
packets_to_send = self.strategy.act_on_packet(packet, self.logger, direction="out")
|
||||
if packets_to_send:
|
||||
self.overhead += (len(packets_to_send) - 1)
|
||||
|
||||
# Send all of the packets we've collected to send
|
||||
for out_packet in packets_to_send:
|
||||
|
@ -270,8 +368,9 @@ class Engine():
|
|||
"""
|
||||
if not self.running_nfqueue:
|
||||
return
|
||||
packet = actions.packet.Packet(IP(nfpacket.get_payload()))
|
||||
packet = layers.packet.Packet(IP(nfpacket.get_payload()))
|
||||
|
||||
if self.save_seen_packets:
|
||||
self.seen_packets.append(packet)
|
||||
|
||||
self.logger.debug("Received packet: %s", str(packet))
|
||||
|
@ -279,7 +378,7 @@ class Engine():
|
|||
# Run the given strategy
|
||||
packets = self.strategy.act_on_packet(packet, self.logger, direction="in")
|
||||
|
||||
# Censors will often send RA packets to disrupt a TCP stream - record this
|
||||
# GFW will send RA packets to disrupt a TCP stream
|
||||
if packet.haslayer("TCP") and packet.get("TCP", "flags") == "RA":
|
||||
self.censorship_detected = True
|
||||
|
||||
|
@ -289,6 +388,11 @@ class Engine():
|
|||
nfpacket.drop()
|
||||
return
|
||||
|
||||
if self.forwarder:
|
||||
nfpacket.drop()
|
||||
self.handle_packet(packet)
|
||||
return
|
||||
|
||||
# Otherwise, overwrite this packet with the packet the action trees gave back
|
||||
nfpacket.set_payload(bytes(packets[0]))
|
||||
|
||||
|
@ -306,13 +410,21 @@ def get_args():
|
|||
"""
|
||||
parser = argparse.ArgumentParser(description='The engine that runs a given strategy.')
|
||||
parser.add_argument('--server-port', type=int, action='store', required=True)
|
||||
parser.add_argument('--server-side', action='store_true', help="If this strategy is running on the server side of a connection")
|
||||
parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test. If not provided, one will be generated.")
|
||||
parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test")
|
||||
parser.add_argument('--sender-ip', action='store', help="IP address of sending machine, used for NAT")
|
||||
parser.add_argument('--routing-ip', action='store', help="Public IP of this machine, used for NAT")
|
||||
parser.add_argument('--forward-ip', action='store', help="IP address to forward traffic to")
|
||||
parser.add_argument('--strategy', action='store', help="Strategy to deploy")
|
||||
parser.add_argument('--output-directory', default="trials", action='store', help="Where to output logs, captures, and results. Defaults to trials/.")
|
||||
parser.add_argument('--forward', action='store_true', help='Enable if this is forwarding traffic')
|
||||
parser.add_argument('--server-side', action='store_true', help='Enable if this is running on the server side')
|
||||
parser.add_argument('--log', action='store', default="debug",
|
||||
choices=("debug", "info", "warning", "critical", "error"),
|
||||
help="Sets the log level")
|
||||
parser.add_argument('--no-save-packets', action='store_false', help='Disables recording captured packets')
|
||||
parser.add_argument("--in-queue-num", action="store", help="NfQueue number for incoming packets", default=1, type=int)
|
||||
parser.add_argument("--out-queue-num", action="store", help="NfQueue number for outgoing packets", default=None, type=int)
|
||||
parser.add_argument("--demo-mode", action='store_true', help="Replaces all IPs with dummy IPs in log messages so as not to reveal sensitive IP addresses")
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
@ -323,12 +435,23 @@ def main(args):
|
|||
Kicks off the engine with the given arguments.
|
||||
"""
|
||||
try:
|
||||
nat_config = {}
|
||||
if args.get("sender_ip") and args.get("routing_ip") and args.get("forward_ip"):
|
||||
nat_config = {"sender_ip" : args["sender_ip"],
|
||||
"routing_ip" : args["routing_ip"],
|
||||
"forward_ip" : args["forward_ip"]}
|
||||
|
||||
eng = Engine(args["server_port"],
|
||||
args["strategy"],
|
||||
environment_id=args.get("environment_id"),
|
||||
output_directory = args.get("output_directory"),
|
||||
server_side=args.get("server_side"),
|
||||
log_level=args["log"])
|
||||
environment_id=args["environment_id"],
|
||||
server_side=args["server_side"],
|
||||
output_directory=args["output_directory"],
|
||||
forwarder=nat_config,
|
||||
log_level=args["log"],
|
||||
in_queue_num=args["in_queue_num"],
|
||||
out_queue_num=args["out_queue_num"],
|
||||
save_seen_packets=args["no_save_packets"],
|
||||
demo_mode=args["demo_mode"])
|
||||
eng.initialize_nfqueue()
|
||||
while True:
|
||||
time.sleep(0.5)
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,835 @@
|
|||
"""
|
||||
Main evolution driver for Geneva (GENetic EVAsion). This file performs the genetic algorithm,
|
||||
and relies on the evaluator (evaluator.py) to provide fitness evaluations of each individual.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import random
|
||||
import subprocess as sp
|
||||
import sys
|
||||
|
||||
import actions.strategy
|
||||
import actions.tree
|
||||
import actions.trigger
|
||||
import evaluator
|
||||
import layers.packet
|
||||
|
||||
# Grab the terminal size for printing
|
||||
try:
|
||||
_, COLUMNS = sp.check_output(['stty', 'size']).decode().split()
|
||||
# If pytest has capturing enabled or this is run without a tty, catch the exception
|
||||
except sp.CalledProcessError:
|
||||
_, COLUMNS = 0, 0
|
||||
|
||||
|
||||
def setup_logger(log_level):
|
||||
"""
|
||||
Sets up the logger. This will log at the specified level to "ga.log" and at debug level to "ga_debug.log".
|
||||
Logs are stored in the trials/ directory under a run-specific folder.
|
||||
Example: trials/2020-01-01_01:00:00/logs/ga.log
|
||||
|
||||
Args:
|
||||
log_level (str): Log level to use in setting up the logger ("debug")
|
||||
"""
|
||||
level = log_level.upper()
|
||||
assert level in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], "Unknown log level %s" % level
|
||||
actions.utils.CONSOLE_LOG_LEVEL = level.lower()
|
||||
|
||||
# Setup needed folders
|
||||
ga_log_dir = actions.utils.setup_dirs(actions.utils.RUN_DIRECTORY)
|
||||
|
||||
ga_log = os.path.join(ga_log_dir, "ga.log")
|
||||
ga_debug_log = os.path.join(ga_log_dir, "ga_debug.log")
|
||||
|
||||
# Configure logging globally
|
||||
formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s:%(message)s', datefmt="%Y-%m-%d %H:%M:%S")
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', datefmt="%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# Set up the root logger
|
||||
logger = logging.getLogger("ga_%s" % actions.utils.RUN_DIRECTORY)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.propagate = False
|
||||
|
||||
setattr(logger, "ga_log_dir", ga_log_dir)
|
||||
|
||||
# If this logger's handlers have already been set up, don't add them again
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
# Set log level of console
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(level)
|
||||
console.setFormatter(formatter)
|
||||
logger.addHandler(console)
|
||||
|
||||
# Add a DEBUG file handler to send all the debug output to a file
|
||||
debug_file_handler = logging.FileHandler(ga_debug_log)
|
||||
debug_file_handler.setFormatter(formatter)
|
||||
debug_file_handler.setLevel(logging.DEBUG)
|
||||
logger.addHandler(debug_file_handler)
|
||||
|
||||
# Add a file handler to send all the output to a file
|
||||
file_handler = logging.FileHandler(ga_log)
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.setLevel(level)
|
||||
logger.addHandler(file_handler)
|
||||
return logger
|
||||
|
||||
|
||||
def collect_plugin_args(cmd, plugin, plugin_type, message=None):
|
||||
"""
|
||||
Collects and prints arguments for a given plugin.
|
||||
|
||||
Args:
|
||||
cmd (list): sys.argv or a list of args to parse
|
||||
plugin (str): Name of plugin to import ("http")
|
||||
plugin_type (str): Component of plugin to import ("client")
|
||||
message (str): message to override for printing
|
||||
"""
|
||||
if not message:
|
||||
message = plugin_type
|
||||
try:
|
||||
_, cls = actions.utils.import_plugin(plugin, plugin_type)
|
||||
print("\n\n")
|
||||
print("=" * int(COLUMNS))
|
||||
print("Options for --test-type %s %s" % (plugin, message))
|
||||
cls.get_args(cmd)
|
||||
# Catch SystemExit here, as this is what argparse raises when --help is passed
|
||||
except (SystemExit, Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_args(cmd):
|
||||
"""
|
||||
Sets up argparse and collects arguments.
|
||||
|
||||
Args:
|
||||
cmd (list): sys.argv or a list of args to parse
|
||||
|
||||
Returns:
|
||||
namespace: Parsed arguments
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description='Genetic algorithm for evolving censorship evasion.\n\nevolve.py uses a pass-through argument system to pass the command line arguments through different files in the system, including the evaluator (evaluator.py) and a given plugin (plugins/). --help will collect all these arguments.', add_help=False, prog="evolve.py")
|
||||
|
||||
parser.add_argument('--test-type', action='store', choices=actions.utils.get_plugins(), default="http", help="plugin to launch")
|
||||
|
||||
# Add help message separately so we can collect the help messages of all of the other parsers
|
||||
parser.add_argument('-h', '--help', action='store_true', default=False, help='print this help message and exit')
|
||||
|
||||
# Control aspects of individuals
|
||||
ind_group = parser.add_argument_group('control aspects of individual strategies')
|
||||
ind_group.add_argument('--in-trees', action='store', type=int, default=0, help='starting # of input-direction action trees per strategy. Disables inbound forest if set to 0')
|
||||
ind_group.add_argument('--out-trees', action='store', type=int, default=1, help='starting # of output-direction action trees per strategy')
|
||||
ind_group.add_argument('--in-actions', action='store', type=int, default=2, help='starting # of input-direction actions per action tree')
|
||||
ind_group.add_argument('--out-actions', action='store', type=int, default=2, help='starting # of output-direction actions per action tree')
|
||||
ind_group.add_argument('--fix-trigger', action='store', help='fix all triggers for this evolution to a given trigger')
|
||||
|
||||
# Options to control the population pool
|
||||
pop_group = parser.add_argument_group('control aspects of the population pool')
|
||||
pop_group.add_argument('--load-from', action='store', help="Load population from a generation file")
|
||||
pop_group.add_argument('--seed', action='store', help='seed strategy to initialize the population to.')
|
||||
|
||||
# Options to evaluate and exit, skip evaluation, and to specify the type of test
|
||||
evaluation_group = parser.add_argument_group('control aspects of strategy evaluation')
|
||||
evaluation_group.add_argument('--eval-only', action='store', default=None, help='only evaluate fitness for a given strategy or file of strategies')
|
||||
evaluation_group.add_argument('--no-eval', action='store_true', help="Disable evaluator for debugging")
|
||||
evaluation_group.add_argument('--runs', action='store', type=int, default=1, help='number of times each strategy should be run for one evaluation (default 1, fitness is averaged).')
|
||||
|
||||
# Hyperparameters for genetic algorithm
|
||||
genetic_group = parser.add_argument_group('control aspects of the genetic algorithm')
|
||||
genetic_group.add_argument('--elite-clones', action='store', type=int, default=3, help='number copies of the highest performing individual that should be propagated to the next generation.')
|
||||
genetic_group.add_argument('--mutation-pb', action='store', type=float, default=0.99, help='mutation probability')
|
||||
genetic_group.add_argument('--crossover-pb', action='store', type=float, default=0.4, help='crossover probability')
|
||||
genetic_group.add_argument('--allowed-retries', action='store', type=int, default=20, help='maximum number of times GA will generate any given individual')
|
||||
genetic_group.add_argument('--generations', type=int, action='store', default=50, help="number of generations to run for.")
|
||||
genetic_group.add_argument('--population', type=int, action='store', default=250, help="size of population.")
|
||||
genetic_group.add_argument('--no-reject-empty', action='store_true', default=False, help="disable mutation rejection of empty strategies")
|
||||
genetic_group.add_argument('--no-canary', action='store_true', help="disable canary phase")
|
||||
|
||||
# Limit access to certain protocols, fields, actions, or types of individuals
|
||||
filter_group = parser.add_argument_group('limit access to certain protocols, fields, actions, or types of individuals')
|
||||
filter_group.add_argument('--protos', action="store", default="TCP", help="allow the GA to scope only to these protocols")
|
||||
filter_group.add_argument('--fields', action='store', default="", help='restrict the GA to only seeing given fields')
|
||||
filter_group.add_argument('--disable-fields', action='store', default="", help='restrict the GA to never using given fields')
|
||||
filter_group.add_argument('--no-gas', action="store_true", help="disables trigger gas")
|
||||
filter_group.add_argument('--disable-action', action='store', default="sleep,trace", help='disables specific actions')
|
||||
|
||||
# Logging
|
||||
logging_group = parser.add_argument_group('control logging')
|
||||
logging_group.add_argument('--log', action='store', default="info", choices=("debug", "info", "warning", "critical", "error"), help="Sets the log level")
|
||||
logging_group.add_argument('--no-print-hall', action='store_true', help="does not print hall of fame at the end")
|
||||
logging_group.add_argument('--graph-trees', action='store_true', default=False, help='graph trees in addition to outputting to screen')
|
||||
|
||||
# Misc
|
||||
usage_group = parser.add_argument_group('misc usage')
|
||||
usage_group.add_argument('--no-lock-file', default=(os.name == "posix"), action='store_true', help="does not use /lock_file.txt")
|
||||
usage_group.add_argument('--force-cleanup', action='store_true', default=False, help='cleans up all docker containers and networks after evolution')
|
||||
|
||||
if not cmd:
|
||||
parser.error("No arguments specified")
|
||||
|
||||
args, _ = parser.parse_known_args(cmd)
|
||||
|
||||
epilog = "See the README.md for usage."
|
||||
# Override the help message to collect the pass through args
|
||||
if args.help:
|
||||
parser.print_help()
|
||||
print(epilog)
|
||||
print("=" * int(COLUMNS))
|
||||
print("\nevolve.py uses a pass-through argument system to evaluator.py and other parts of Geneva. These arguments are below.\n\n")
|
||||
evaluator.get_arg_parser(cmd).print_help()
|
||||
if args.test_type:
|
||||
collect_plugin_args(cmd, args.test_type, "plugin", message="parent plugin")
|
||||
collect_plugin_args(cmd, args.test_type, "client")
|
||||
collect_plugin_args(cmd, args.test_type, "server")
|
||||
raise SystemExit
|
||||
return args
|
||||
|
||||
|
||||
def fitness_function(logger, population, ga_evaluator):
|
||||
"""
|
||||
Calls the evaluator to evaluate a given population of strategies.
|
||||
Sets the .fitness attribute of each individual.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
population (list): List of individuals to evaluate
|
||||
ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator object to evaluate with
|
||||
|
||||
Returns:
|
||||
list: Population post-evaluation
|
||||
"""
|
||||
if ga_evaluator:
|
||||
return ga_evaluator.evaluate(population)
|
||||
|
||||
for ind in population:
|
||||
ind.fitness = 0
|
||||
logger.info("[%s] Fitness %d: %s", -1, ind.fitness, str(ind))
|
||||
|
||||
return population
|
||||
|
||||
|
||||
def sel_random(individuals, k):
|
||||
"""
|
||||
Implementation credit to DEAP: https://github.com/DEAP/deap
|
||||
Select *k* individuals at random from the input *individuals* with
|
||||
replacement. The list returned contains references to the input
|
||||
*individuals*.
|
||||
|
||||
Args:
|
||||
individuals (list): A list of individuals to select from.
|
||||
k (int): The number of individuals to select.
|
||||
|
||||
Returns:
|
||||
list: A list of selected individuals.
|
||||
"""
|
||||
return [random.choice(individuals) for _ in range(k)]
|
||||
|
||||
|
||||
def selection_tournament(individuals, k, tournsize, fit_attr="fitness"):
|
||||
"""
|
||||
Implementation credit to DEAP: https://github.com/DEAP/deap
|
||||
Select the best individual among *tournsize* randomly chosen
|
||||
individuals, *k* times. The list returned contains
|
||||
references to the input *individuals*.
|
||||
|
||||
Args:
|
||||
individuals (list): A list of individuals to select from.
|
||||
k (int): The number of individuals to select.
|
||||
tournsize (int): The number of individuals participating in each tournament.
|
||||
fit_attr: The attribute of individuals to use as selection criterion (defaults to "fitness")
|
||||
|
||||
Returns:
|
||||
list: A list of selected individuals.
|
||||
"""
|
||||
chosen = []
|
||||
for _ in range(k):
|
||||
aspirants = sel_random(individuals, tournsize)
|
||||
chosen.append(copy.deepcopy(max(aspirants, key=operator.attrgetter(fit_attr))))
|
||||
return chosen
|
||||
|
||||
|
||||
def get_unique_population_size(population):
|
||||
"""
|
||||
Computes number of unique individuals in a population.
|
||||
|
||||
Args:
|
||||
population (list): Population list
|
||||
"""
|
||||
uniques = {}
|
||||
for ind in population:
|
||||
uniques[str(ind)] = True
|
||||
return len(list(uniques.keys()))
|
||||
|
||||
|
||||
def add_to_hof(hof, population):
|
||||
"""
|
||||
Iterates over the current population and updates the hall of fame.
|
||||
The hall of fame is a dictionary that tracks the fitness of every
|
||||
run of every strategy ever.
|
||||
|
||||
Args:
|
||||
hof (dict): Current hall of fame
|
||||
population (list): Population list
|
||||
|
||||
Returns:
|
||||
dict: Updated hall of fame
|
||||
"""
|
||||
for ind in population:
|
||||
if str(ind) not in hof:
|
||||
hof[str(ind)] = []
|
||||
hof[str(ind)].append(ind.fitness)
|
||||
|
||||
return hof
|
||||
|
||||
|
||||
def generate_strategy(logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, environment_id=None, disabled=None):
|
||||
"""
|
||||
Generates a strategy individual.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
num_in_trees (int): Number of trees to initialize in the inbound forest
|
||||
num_out_trees (int): Number of trees to initialize in the outbound forest
|
||||
num_in_actions (int): Number of actions to initialize in the each inbound tree
|
||||
num_out_actions (int): Number of actions to initialize in the each outbound tree
|
||||
environment_id (str, optional): Environment ID to assign to the new individual
|
||||
disabled (str, optional): List of actions that should not be considered in building a new strategy
|
||||
|
||||
Returns:
|
||||
:obj:`actions.strategy.Strategy`: A new strategy object
|
||||
"""
|
||||
try:
|
||||
strat = actions.strategy.Strategy([], [], environment_id=environment_id)
|
||||
strat.initialize(logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, disabled=disabled)
|
||||
except Exception:
|
||||
logger.exception("Failure to generate strategy")
|
||||
raise
|
||||
|
||||
return strat
|
||||
|
||||
|
||||
def mutation_crossover(logger, population, hall, options):
|
||||
"""
|
||||
Apply crossover and mutation on the offspring.
|
||||
|
||||
Hall is a copy of the hall of fame, used to accept or reject mutations.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
population (list): Population of individuals
|
||||
hall (dict): Current hall of fame
|
||||
options (dict): Options to override settings. Accepted keys are:
|
||||
"crossover_pb" (float): probability of crossover
|
||||
"mutation_pb" (float): probability of mutation
|
||||
"allowed_retries" (int): number of times a strategy is allowed to exist in the hall of fame.
|
||||
"no_reject_empty" (bool): whether or not empty strategies should be rejected
|
||||
|
||||
Returns:
|
||||
list: New population after mutation
|
||||
"""
|
||||
cxpb = options.get("crossover_pb", 0.5)
|
||||
mutpb = options.get("mutation_pb", 0.5)
|
||||
|
||||
offspring = copy.deepcopy(population)
|
||||
for i in range(1, len(offspring), 2):
|
||||
if random.random() < cxpb:
|
||||
ind = offspring[i - 1]
|
||||
actions.strategy.mate(ind, offspring[i], indpb=0.5)
|
||||
offspring[i - 1].fitness, offspring[i].fitness = -1000, -1000
|
||||
|
||||
for i in range(len(offspring)):
|
||||
if random.random() < mutpb:
|
||||
|
||||
mutation_accepted = False
|
||||
while not mutation_accepted:
|
||||
test_subject = copy.deepcopy(offspring[i])
|
||||
mutate_individual(logger, test_subject)
|
||||
|
||||
# Pull out some metadata about this proposed mutation
|
||||
fitness_history = hall.get(str(test_subject), [])
|
||||
|
||||
# If we've seen this strategy 10 times before and it has always failed,
|
||||
# or if we have seen it 20 times already, or if it is an empty strategy,
|
||||
# reject this mutation and get another
|
||||
if len(fitness_history) >= 10 and all(fitness < 0 for fitness in fitness_history) or \
|
||||
len(fitness_history) >= options.get("allowed_retries", 20) or \
|
||||
(len(test_subject) == 0 and not options.get("no_reject_empty")):
|
||||
mutation_accepted = False
|
||||
else:
|
||||
mutation_accepted = True
|
||||
|
||||
offspring[i] = test_subject
|
||||
offspring[i].fitness = -1000
|
||||
|
||||
return offspring
|
||||
|
||||
|
||||
def mutate_individual(logger, ind):
|
||||
"""
|
||||
Simply calls the mutate function of the given individual.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
ind (:obj:`actions.strategy.Strategy`): A strategy individual to mutate
|
||||
|
||||
Returns:
|
||||
:obj:`actions.strategy.Strategy`: Mutated individual
|
||||
"""
|
||||
return ind.mutate(logger)
|
||||
|
||||
|
||||
def run_collection_phase(logger, ga_evaluator):
|
||||
"""Individual mutation works best when it has seen real packets to base
|
||||
action and trigger values off of, instead of blindly fuzzing packets.
|
||||
Usually, the 0th generation is useless because it hasn't seen any real
|
||||
packets yet, and it bases everything off fuzzed data. To combat this, a
|
||||
canary phase is done instead.
|
||||
|
||||
In the canary phase, a single dummy individual is evaluated to capture
|
||||
packets. Once the packets are captured, they are associated with all of the
|
||||
initial population pool, so all of the individuals have some packets to base
|
||||
their data off of.
|
||||
|
||||
Since this phase by necessity requires the evaluator, this is only run if
|
||||
--no-eval is not specified.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator object to evaluate with
|
||||
|
||||
Returns:
|
||||
str: ID of the test 'canary' strategy evaluated to do initial collection
|
||||
"""
|
||||
canary = generate_strategy(logger, 0, 0, 0, 0, None, disabled=[])
|
||||
canary_id = ga_evaluator.canary_phase(canary)
|
||||
if not canary_id:
|
||||
return []
|
||||
return canary_id
|
||||
|
||||
|
||||
def write_generation(filename, population):
|
||||
"""
|
||||
Writes the population pool for a specific generation.
|
||||
|
||||
Args:
|
||||
filename (str): Name of file to write the generation out to
|
||||
population (list): List of individuals to write out
|
||||
"""
|
||||
# Open File as writable
|
||||
with open(filename, "w") as fd:
|
||||
# Write each individual to file
|
||||
for index, individual in enumerate(population):
|
||||
if index == len(population) - 1:
|
||||
fd.write(str(individual))
|
||||
else:
|
||||
fd.write(str(individual) + "\n")
|
||||
|
||||
|
||||
def load_generation(logger, filename):
|
||||
"""
|
||||
Loads strategies from a file
|
||||
|
||||
Args:
|
||||
logger (:obj:`logger.Logger`): A logger to log with
|
||||
filename (str): Filename of file containing newline separated strategies
|
||||
to read generation from
|
||||
"""
|
||||
population = []
|
||||
with open(filename) as file:
|
||||
|
||||
# Read each individual from file
|
||||
for individual in file:
|
||||
strategy = actions.utils.parse(individual, logger)
|
||||
population.append(strategy)
|
||||
|
||||
return population
|
||||
|
||||
|
||||
def initialize_population(logger, options, canary_id, disabled=None):
|
||||
"""
|
||||
Initializes the population from either random strategies or strategies
|
||||
located in a file.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
options (dict): Options to respect in generating initial population.
|
||||
Options that can be specified as keys:
|
||||
|
||||
"load_from" (str, optional): File to load population from
|
||||
population_size (int): Size of population to initialize
|
||||
|
||||
"in-trees" (int): Number of trees to initialize in inbound forest
|
||||
of each individual
|
||||
|
||||
"out-trees" (int): Number of trees to initialize in outbound forest
|
||||
of each individual
|
||||
|
||||
"in-actions" (int): Number of actions to initialize in each
|
||||
inbound tree of each individual
|
||||
|
||||
"out-actions" (int): Number of actions to initialize in each
|
||||
outbound tree of each individual
|
||||
|
||||
"seed" (str): Strategy to seed this pool with
|
||||
canary_id (str): ID of the canary strategy, used to associate each new
|
||||
strategy with the packets captured during the canary phase
|
||||
disabled (list, optional): List of actions that are disabled
|
||||
|
||||
Returns:
|
||||
list: New population of individuals
|
||||
"""
|
||||
|
||||
if options.get("load_from"):
|
||||
# Load the population from a file
|
||||
return load_generation(logger, options["load_from"])
|
||||
|
||||
# Generate random strategies
|
||||
population = []
|
||||
|
||||
for _ in range(options["population_size"]):
|
||||
p = generate_strategy(logger, options["in-trees"], options["out-trees"], options["in-actions"],
|
||||
options["out-actions"], options["seed"], environment_id=canary_id,
|
||||
disabled=disabled)
|
||||
population.append(p)
|
||||
|
||||
return population
|
||||
|
||||
|
||||
def genetic_solve(logger, options, ga_evaluator):
|
||||
"""
|
||||
Run genetic algorithm with given options.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
options (dict): Options to respect.
|
||||
ga_evaluator (:obj:`evaluator.Evaluator`): Evaluator to evaluate
|
||||
strategies with
|
||||
|
||||
Returns:
|
||||
dict: Hall of fame of individuals
|
||||
"""
|
||||
# Directory to save off each generation so evolution can be resumed
|
||||
ga_generations_dir = os.path.join(actions.utils.RUN_DIRECTORY, "generations")
|
||||
|
||||
hall = {}
|
||||
canary_id = None
|
||||
if ga_evaluator and not options["no-canary"]:
|
||||
canary_id = run_collection_phase(logger, ga_evaluator)
|
||||
else:
|
||||
logger.info("Skipping initial collection phase.")
|
||||
|
||||
population = initialize_population(logger, options, canary_id, disabled=options["disable_action"])
|
||||
|
||||
try:
|
||||
offspring = []
|
||||
elite_clones = []
|
||||
if options["seed"]:
|
||||
elite_clones = [actions.utils.parse(options["seed"], logger)]
|
||||
|
||||
# Evolution over given number of generations
|
||||
for gen in range(options["num_generations"]):
|
||||
# Debug printing
|
||||
logger.info("="*(int(COLUMNS) - 25))
|
||||
logger.info("Generation %d:", gen)
|
||||
|
||||
# Save current population pool
|
||||
filename = os.path.join(ga_generations_dir, "generation" + str(gen) + ".txt")
|
||||
write_generation(filename, population)
|
||||
|
||||
# To store the best individuals of this generation to print
|
||||
best_fit, best_ind = -10000, None
|
||||
|
||||
# Mutation and crossover
|
||||
offspring = mutation_crossover(logger, population, hall, options)
|
||||
offspring += elite_clones
|
||||
|
||||
# Calculate fitness
|
||||
offspring = fitness_function(logger, offspring, ga_evaluator)
|
||||
|
||||
total_fitness = 0
|
||||
# Iterate over the offspring to find the best individual for printing
|
||||
for ind in offspring:
|
||||
if ind.fitness is None and ga_evaluator:
|
||||
logger.error("No fitness for individual found: %s.", str(ind))
|
||||
continue
|
||||
total_fitness += ind.fitness
|
||||
if ind.fitness is not None and ind.fitness >= best_fit:
|
||||
best_fit = ind.fitness
|
||||
best_ind = ind
|
||||
|
||||
# Check if any individuals of this generation belong in the hall of fame
|
||||
hall = add_to_hof(hall, offspring)
|
||||
|
||||
# Save current hall of fame
|
||||
filename = os.path.join(ga_generations_dir, "hall" + str(gen) + ".txt")
|
||||
write_hall(filename, hall)
|
||||
|
||||
# Status printing for this generation
|
||||
logger.info("\nGeneration: %d | Unique Inviduals: %d | Avg Fitness: %d | Best Fitness [%s] %s: %s",
|
||||
gen, get_unique_population_size(population), round(total_fitness / float(len(offspring)), 2),
|
||||
best_ind.environment_id, str(best_fit), str(best_ind))
|
||||
|
||||
# Select next generation
|
||||
population = selection_tournament(offspring, k=len(population) - options["elite_clones"], tournsize=10)
|
||||
|
||||
# Add the elite clones
|
||||
if options["elite_clones"] > 0:
|
||||
elite_clones = [copy.deepcopy(best_ind) for x in range(options["elite_clones"])]
|
||||
|
||||
# If the user interrupted, try to gracefully shutdown
|
||||
except KeyboardInterrupt:
|
||||
# Only need to stop the evaluator if one is defined
|
||||
if ga_evaluator:
|
||||
ga_evaluator.stop = True
|
||||
logger.info("")
|
||||
|
||||
finally:
|
||||
if options["force_cleanup"]:
|
||||
# Try to clean up any hanging docker containers/networks from the run
|
||||
logger.warning("Cleaning up docker...")
|
||||
try:
|
||||
sp.check_call("docker stop $(docker ps -aq) > /dev/null 2>&1", shell=True)
|
||||
except sp.CalledProcessError:
|
||||
pass
|
||||
|
||||
return hall
|
||||
|
||||
|
||||
def collect_results(hall_of_fame):
|
||||
"""
|
||||
Collect final results from offspring.
|
||||
|
||||
Args:
|
||||
hall_of_fame (dict): Hall of fame of individuals
|
||||
|
||||
Returns:
|
||||
str: Formatted printout of the hall of fame
|
||||
"""
|
||||
# Sort first on number of runs, then by fitness.
|
||||
best_inds = sorted(hall_of_fame, key=lambda ind: (len(hall_of_fame[ind]), sum(hall_of_fame[ind])/len(hall_of_fame[ind])))
|
||||
output = "Results: \n"
|
||||
for ind in best_inds:
|
||||
sind = str(ind)
|
||||
output += "Avg. Fitness %s: %s (Evaluated %d times: %s)\n" % (sum(hall_of_fame[sind])/len(hall_of_fame[sind]), sind, len(hall_of_fame[sind]), hall_of_fame[sind])
|
||||
return output
|
||||
|
||||
|
||||
def print_results(hall_of_fame, logger):
|
||||
"""
|
||||
Prints hall of fame.
|
||||
|
||||
Args:
|
||||
hall_of_fame (dict): Hall of fame to print
|
||||
logger (:obj:`logging.Logger`): A logger to log results with
|
||||
"""
|
||||
logger.info("\n%s", collect_results(hall_of_fame))
|
||||
|
||||
|
||||
def write_hall(filename, hall_of_fame):
|
||||
"""
|
||||
Writes hall of fame out to a file.
|
||||
|
||||
Args:
|
||||
filename (str): Filename to write results to
|
||||
hall_of_fame (dict): Hall of fame of individuals
|
||||
"""
|
||||
with open(filename, "w") as fd:
|
||||
fd.write(collect_results(hall_of_fame))
|
||||
|
||||
|
||||
def eval_only(logger, requested, ga_evaluator, runs=1):
|
||||
"""
|
||||
Parses a string representation of a given strategy and runs it
|
||||
through the evaluator.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
requested (str): String representation of requested strategy or filename
|
||||
of strategies
|
||||
ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator to evaluate with
|
||||
runs (int): Number of times each strategy should be evaluated
|
||||
|
||||
Returns:
|
||||
float: Success rate of tested strategies
|
||||
"""
|
||||
# The user can specify a file that contains strategies - check first if that is the case
|
||||
if os.path.isfile(requested):
|
||||
with open(requested, "r") as fd:
|
||||
requested_strategies = fd.readlines()
|
||||
if not requested_strategies:
|
||||
logger.error("No strategies found in %s", requested)
|
||||
return None
|
||||
else:
|
||||
requested_strategies = [requested]
|
||||
# We want to override the client's default strategy retry logic to ensure
|
||||
# we test to the number of runs requested
|
||||
ga_evaluator.runs = 1
|
||||
population = []
|
||||
|
||||
for requested in requested_strategies:
|
||||
for i in range(runs):
|
||||
ind = actions.utils.parse(requested, logger)
|
||||
population.append(ind)
|
||||
logging.info("Computing fitness for: %s", str(ind))
|
||||
logging.info("\n%s", ind.pretty_print())
|
||||
|
||||
fits = []
|
||||
success = 0
|
||||
# Once the population has been parsed and built, test it
|
||||
fitness_function(logger, population, ga_evaluator)
|
||||
for strat in population:
|
||||
fits.append(strat.fitness)
|
||||
i = 0
|
||||
logger.info(fits)
|
||||
for fitness in fits:
|
||||
if fitness > 0:
|
||||
success += 1
|
||||
logger.info("Trial %d: success! (fitness = %d)", i, fitness)
|
||||
else:
|
||||
logger.info("Trial %d: failure! (fitness = %d)", i, fitness)
|
||||
i += 1
|
||||
if fits:
|
||||
logger.info("Overall %d/%d = %d%%", success, i, int((float(success)/float(i)) * 100))
|
||||
logger.info("Exiting eval-only.")
|
||||
return float(success)/float(i)
|
||||
|
||||
|
||||
def restrict_headers(logger, protos, filter_fields, disabled_fields):
|
||||
"""
|
||||
Restricts which protocols/fields can be accessed by the algorithm.
|
||||
|
||||
Args:
|
||||
logger (:obj:`logging.Logger`): A logger to log with
|
||||
protos (str): Comma separated string of protocols that are allowed
|
||||
filter_fields (str): Comma separated string of fields to allow
|
||||
disabled_fields (str): Comma separated string of fields to disable
|
||||
"""
|
||||
# Retrieve flag and protocol filters, and validate them
|
||||
protos = protos.upper().split(",")
|
||||
if filter_fields:
|
||||
filter_fields = filter_fields.lower().split(",")
|
||||
if disabled_fields:
|
||||
disabled_fields = disabled_fields.split(",")
|
||||
|
||||
layers.packet.Packet.restrict_fields(logger, protos, filter_fields, disabled_fields)
|
||||
|
||||
|
||||
def driver(cmd):
|
||||
"""
|
||||
Main workflow driver for the solver. Parses flags and input data, and initiates solving.
|
||||
|
||||
Args:
|
||||
cmd (list): sys.argv or a list of arguments
|
||||
|
||||
Returns:
|
||||
dict: Hall of fame of individuals
|
||||
"""
|
||||
# Parse the given arguments
|
||||
args = get_args(cmd)
|
||||
|
||||
logger = setup_logger(args.log)
|
||||
|
||||
lock_file_path = "/lock_file.txt"
|
||||
if not args.no_lock_file and os.path.exists(lock_file_path):
|
||||
logger.info("Lock file \"%s\" already exists.", lock_file_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
if not args.no_lock_file:
|
||||
# Create lock file to prevent interference between multiple runs
|
||||
open(lock_file_path, "w+")
|
||||
|
||||
# Log the command run
|
||||
logger.debug("Launching strategy evolution: %s", " ".join(cmd))
|
||||
logger.info("Logging results to %s", logger.ga_log_dir)
|
||||
|
||||
if args.no_eval and args.eval_only:
|
||||
print("ERROR: Cannot --eval-only with --no-eval.")
|
||||
return None
|
||||
|
||||
requested_strategy = args.eval_only
|
||||
|
||||
# Define an evaluator for this session
|
||||
ga_evaluator = None
|
||||
if not args.no_eval:
|
||||
cmd += ["--output-directory", actions.utils.RUN_DIRECTORY]
|
||||
ga_evaluator = evaluator.Evaluator(cmd, logger)
|
||||
|
||||
# Check if the user only wanted to evaluate a single given strategy
|
||||
# If so, evaluate it, and exit
|
||||
if requested_strategy or requested_strategy == "":
|
||||
# Disable evaluator empty strategy skipping
|
||||
ga_evaluator.skip_empty = False
|
||||
eval_only(logger, requested_strategy, ga_evaluator, runs=args.runs)
|
||||
return None
|
||||
|
||||
restrict_headers(logger, args.protos, args.fields, args.disable_fields)
|
||||
|
||||
actions.trigger.GAS_ENABLED = (not args.no_gas)
|
||||
if args.fix_trigger:
|
||||
actions.trigger.FIXED_TRIGGER = actions.trigger.Trigger.parse(args.fix_trigger)
|
||||
|
||||
requested_seed = args.seed
|
||||
if requested_seed or requested_seed == "":
|
||||
try:
|
||||
requested_seed = actions.utils.parse(args.seed, logger)
|
||||
except (TypeError, AssertionError, actions.tree.ActionTreeParseError):
|
||||
logger.error("Failed to parse given strategy: %s", requested_seed)
|
||||
raise
|
||||
|
||||
# Record all of the options supplied by the user to pass to the GA
|
||||
options = {}
|
||||
options["no_reject_empty"] = not args.no_reject_empty
|
||||
options["population_size"] = args.population
|
||||
options["out-trees"] = args.out_trees
|
||||
options["in-trees"] = args.in_trees
|
||||
options["in-actions"] = args.in_actions
|
||||
options["out-actions"] = args.out_actions
|
||||
options["force_cleanup"] = args.force_cleanup
|
||||
options["num_generations"] = args.generations
|
||||
options["seed"] = args.seed
|
||||
options["elite_clones"] = args.elite_clones
|
||||
options["allowed_retries"] = args.allowed_retries
|
||||
options["mutation_pb"] = args.mutation_pb
|
||||
options["crossover_pb"] = args.crossover_pb
|
||||
options["no-canary"] = args.no_canary
|
||||
options["load_from"] = args.load_from
|
||||
|
||||
disable_action = []
|
||||
if args.disable_action:
|
||||
disable_action = args.disable_action.split(",")
|
||||
options["disable_action"] = disable_action
|
||||
|
||||
logger.info("Initializing %d strategies with %d input-action trees and %d output-action trees of input size %d and output size %d for evolution over %d generations.",
|
||||
args.population, args.in_trees, args.out_trees, args.in_actions, args.out_actions, args.generations)
|
||||
|
||||
hall_of_fame = {}
|
||||
try:
|
||||
# Kick off the main genetic driver
|
||||
hall_of_fame = genetic_solve(logger, options, ga_evaluator)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("User shutdown requested.")
|
||||
if ga_evaluator:
|
||||
ga_evaluator.shutdown()
|
||||
|
||||
if hall_of_fame and not args.no_print_hall:
|
||||
# Print the final results
|
||||
print_results(hall_of_fame, logger)
|
||||
|
||||
# Teardown the evaluator if needed
|
||||
if ga_evaluator:
|
||||
ga_evaluator.shutdown()
|
||||
finally:
|
||||
# Remove lock file
|
||||
if os.path.exists(lock_file_path):
|
||||
os.remove(lock_file_path)
|
||||
return hall_of_fame
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
driver(sys.argv[1:])
|
|
@ -0,0 +1,189 @@
|
|||
import random
|
||||
from layers.layer import Layer
|
||||
from scapy.all import DNS
|
||||
|
||||
class DNSLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access DNS header fields.
|
||||
"""
|
||||
name = "DNS"
|
||||
protocol = DNS
|
||||
_fields = [
|
||||
"id",
|
||||
"qr",
|
||||
"opcode",
|
||||
"aa",
|
||||
"tc",
|
||||
"rd",
|
||||
"ra",
|
||||
"z",
|
||||
"ad",
|
||||
"cd",
|
||||
"qd",
|
||||
"rcode",
|
||||
"qdcount",
|
||||
"ancount",
|
||||
"nscount",
|
||||
"arcount"
|
||||
]
|
||||
fields = _fields
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the DNS layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
|
||||
self.getters = {
|
||||
"qr" : self.get_bitfield,
|
||||
"aa" : self.get_bitfield,
|
||||
"tc" : self.get_bitfield,
|
||||
"rd" : self.get_bitfield,
|
||||
"ra" : self.get_bitfield,
|
||||
"z" : self.get_bitfield,
|
||||
"ad" : self.get_bitfield,
|
||||
"cd" : self.get_bitfield
|
||||
}
|
||||
|
||||
self.setters = {
|
||||
"qr" : self.set_bitfield,
|
||||
"aa" : self.set_bitfield,
|
||||
"tc" : self.set_bitfield,
|
||||
"rd" : self.set_bitfield,
|
||||
"ra" : self.set_bitfield,
|
||||
"z" : self.set_bitfield,
|
||||
"ad" : self.set_bitfield,
|
||||
"cd" : self.set_bitfield
|
||||
}
|
||||
|
||||
self.generators = {
|
||||
"id" : self.gen_id,
|
||||
"qr" : self.gen_bitfield,
|
||||
"opcode" : self.gen_opcode,
|
||||
"aa" : self.gen_bitfield,
|
||||
"tc" : self.gen_bitfield,
|
||||
"rd" : self.gen_bitfield,
|
||||
"ra" : self.gen_bitfield,
|
||||
"z" : self.gen_bitfield,
|
||||
"ad" : self.gen_bitfield,
|
||||
"cd" : self.gen_bitfield,
|
||||
"rcode" : self.gen_rcode,
|
||||
"qdcount" : self.gen_count,
|
||||
"ancount" : self.gen_count,
|
||||
"nscount" : self.gen_count,
|
||||
"arcount" : self.gen_count
|
||||
}
|
||||
|
||||
def get_bitfield(self, field):
|
||||
""""""
|
||||
return int(getattr(self.layer, field))
|
||||
|
||||
def set_bitfield(self, packet, field, value):
|
||||
""""""
|
||||
return setattr(self.layer, field, int(value))
|
||||
|
||||
def gen_bitfield(self, field):
|
||||
""""""
|
||||
return random.choice([0,1])
|
||||
|
||||
def gen_id(self, field):
|
||||
return random.randint(0, 65535)
|
||||
|
||||
def gen_opcode(self, field):
|
||||
return random.randint(0, 15)
|
||||
|
||||
def gen_rcode(self, field):
|
||||
return random.randint(0, 15)
|
||||
|
||||
def gen_count(self, field):
|
||||
return random.randint(0, 65535)
|
||||
|
||||
@staticmethod
|
||||
def dns_decompress(packet, logger):
|
||||
"""
|
||||
Performs DNS decompression on the given scapy packet, if applicable.
|
||||
Note that DNS compression/decompression must be done on the boundaries
|
||||
of a label, so DNS compression does not support arbitrary offsets.
|
||||
"""
|
||||
# If this is a TCP packet
|
||||
if packet.haslayer("TCP"):
|
||||
raise NotImplementedError
|
||||
|
||||
# Perform no action if this is not a DNS or DNSRQ packet
|
||||
if not packet.haslayer("DNS") or not packet.haslayer("DNSQR"):
|
||||
return packet
|
||||
|
||||
# Extract the query from the DNSQR layer
|
||||
query = packet["DNSQR"].qname.decode()
|
||||
if query[len(query) - 1] != '.':
|
||||
query += '.'
|
||||
|
||||
# Split the query by label
|
||||
labels = query.split(".")
|
||||
|
||||
# Collect the first and second half of the query
|
||||
fhalf = labels[0]
|
||||
shalf = ".".join(labels[1:])
|
||||
|
||||
# Build the first DNS query directly. The format of this a byte string like this:
|
||||
# b'\x07minghui\xc0\x1a\x00\x01\x00\x01'
|
||||
# \x07 = the length of the label in this DNSQR
|
||||
# minghui = the portion of the domain we will request in the first DNSQR
|
||||
# \xc0\x1a = offset into the DNS packet where the rest of the query will be. The actual offset
|
||||
# here is the \x1a - DNS mandates that if compression is used, the first two bits be 11
|
||||
# to differentiate them from the rest. \x1A = 26, which is the length of the DNS header
|
||||
# plus the length of this DNSQR.
|
||||
# \x00\x01 = type A record
|
||||
# \x00\x01 = IN
|
||||
length = bytes([len(fhalf)])
|
||||
label = fhalf.encode()
|
||||
|
||||
# Since the domain will include an extra ".", add 1
|
||||
# 2 * 6 is the DNS header
|
||||
# 1 is the byte that determines the length of the label
|
||||
# len(label) is the length of the label
|
||||
# 2 is the offset pointer
|
||||
# 4 - other record information (class, IN)
|
||||
packet_offset = 2 * 6 + 1 + len(label) + 2 + 2 + 2
|
||||
|
||||
# The word must start with binary 11, so OR the offset with 0xC000.
|
||||
offset = (0xc000 | packet_offset).to_bytes(2, byteorder='big')
|
||||
request = b'\x00\x01\x00\x01'
|
||||
|
||||
dns_qr1 = length + label + offset + request
|
||||
|
||||
# Build the second DNS query directly. The format of the byte string is the same as above
|
||||
# b'\x02ca\x00\x00\x01\x00\x01'
|
||||
# \x02 = length of the remaining domain
|
||||
# ca = portion of the domain in this DNSQR
|
||||
# \x00 = null byte to signify the end of the query
|
||||
# \x00\x01 = type A record
|
||||
# \x00\x01 = IN
|
||||
# Since the second half could potentially contain many labels, this is done in a list comprehension
|
||||
dns_qr2 = b"".join([bytes([len(tld)]) + tld.encode() for tld in shalf.split(".")]) + b"\x00\x01\x00\x01"
|
||||
|
||||
# Next, we must rebuild the DNS packet itself. If we try to have scapy parse either dns_qr1 or dns_qr2, they
|
||||
# will look malformed, since neither contains a complete request. Therefore, we must build the entire
|
||||
# DNS packet at once. First, we must remove the original DNSQR, since this contains the original request
|
||||
del packet["DNS"].qd
|
||||
|
||||
# Once the DNSQR is removed, scapy automatically sets the qdcount to 0. Adjust it to 2
|
||||
packet["DNS"].qdcount = 2
|
||||
|
||||
# Extract the DNS header standalone now for building
|
||||
dns_header = bytes(packet["DNS"])
|
||||
|
||||
dns_packet = DNS(dns_header + dns_qr1 + dns_qr2)
|
||||
|
||||
del packet["DNS"]
|
||||
packet = packet / dns_packet
|
||||
|
||||
# Since the size and data of the packet have changed, force scapy to recalculate the important fields
|
||||
# in below layers, if applicable
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].chksum
|
||||
del packet["IP"].len
|
||||
if packet.haslayer("UDP"):
|
||||
del packet["UDP"].chksum
|
||||
del packet["UDP"].len
|
||||
|
||||
return packet
|
|
@ -0,0 +1,48 @@
|
|||
from layers.layer import Layer
|
||||
from scapy.all import DNSQR
|
||||
|
||||
class DNSQRLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access DNSQR header fields.
|
||||
"""
|
||||
name = "DNSQR"
|
||||
protocol = DNSQR
|
||||
_fields = [
|
||||
"qname",
|
||||
"qtype",
|
||||
"qclass"
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the DNS layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
"qname" : self.get_qname
|
||||
}
|
||||
self.generators = {
|
||||
"qname" : self.gen_qname
|
||||
}
|
||||
|
||||
def get_qname(self, field):
|
||||
"""
|
||||
Returns decoded qname from packet.
|
||||
"""
|
||||
return self.layer.qname.decode('utf-8')
|
||||
|
||||
def gen_qname(self, field):
|
||||
"""
|
||||
Generates domain name.
|
||||
"""
|
||||
return "example.com."
|
||||
|
||||
@classmethod
|
||||
def name_matches(cls, name):
|
||||
"""
|
||||
Scapy returns the name of DNSQR as _both_ DNSQR and "DNS Question Record",
|
||||
which breaks parsing. Override the name_matches method to handle that case
|
||||
here.
|
||||
"""
|
||||
return name.upper() in ["DNSQR", "DNS QUESTION RECORD"]
|
|
@ -0,0 +1,103 @@
|
|||
import random
|
||||
|
||||
from layers.layer import Layer
|
||||
from scapy.all import IP, fuzz, RandIP
|
||||
|
||||
class IPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access IP header fields.
|
||||
"""
|
||||
name = "IP"
|
||||
protocol = IP
|
||||
_fields = [
|
||||
'version',
|
||||
'ihl',
|
||||
'tos',
|
||||
'len',
|
||||
'id',
|
||||
'flags',
|
||||
'frag',
|
||||
'ttl',
|
||||
'proto',
|
||||
'chksum',
|
||||
'src',
|
||||
'dst',
|
||||
'load'
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the IP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
"flags" : self.get_flags,
|
||||
"load" : self.get_load
|
||||
}
|
||||
self.setters = {
|
||||
"flags" : self.set_flags,
|
||||
"load" : self.set_load
|
||||
}
|
||||
self.generators = {
|
||||
"src" : self.gen_ip,
|
||||
"dst" : self.gen_ip,
|
||||
"chksum" : self.gen_chksum,
|
||||
"len" : self.gen_len,
|
||||
"load" : self.gen_load,
|
||||
"flags" : self.gen_flags
|
||||
}
|
||||
|
||||
def gen_len(self, field):
|
||||
"""
|
||||
Generates a valid IP length. Scapy breaks if the length is set to 0, so
|
||||
return a random int starting at 1.
|
||||
"""
|
||||
return random.randint(1, 500)
|
||||
|
||||
def gen_chksum(self, field):
|
||||
"""
|
||||
Generates a checksum.
|
||||
"""
|
||||
return random.randint(1, 65535)
|
||||
|
||||
def gen_ip(self, field):
|
||||
"""
|
||||
Generates an IP address.
|
||||
"""
|
||||
return RandIP()._fix()
|
||||
|
||||
def get_flags(self, field):
|
||||
"""
|
||||
Retrieves flags as a string.
|
||||
"""
|
||||
return str(self.layer.flags)
|
||||
|
||||
def set_flags(self, packet, field, value):
|
||||
"""
|
||||
Sets the flags field. There is a bug in scapy, if you retrieve an empty
|
||||
flags field, it will return "", but you cannot set this value back.
|
||||
To reproduce this bug:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> setattr(IP(), "flags", str(IP().flags)) # raises a ValueError
|
||||
|
||||
To handle this case, this method converts empty string to zero so that
|
||||
it can be safely stored.
|
||||
"""
|
||||
if value == "":
|
||||
value = 0
|
||||
self.layer.flags = value
|
||||
|
||||
def gen_flags(self, field):
|
||||
"""
|
||||
Generates random valid flags.
|
||||
"""
|
||||
sample = fuzz(self.protocol())
|
||||
|
||||
# Since scapy lazily evaluates fuzzing, we first must set a
|
||||
# legitimate value for scapy to evaluate what combination of flags it is
|
||||
sample.flags = sample.flags
|
||||
|
||||
return str(sample.flags)
|
|
@ -0,0 +1,197 @@
|
|||
import binascii
|
||||
import copy
|
||||
import random
|
||||
import string
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
from scapy.all import IP, RandIP, UDP, DNS, DNSQR, Raw, TCP, fuzz
|
||||
|
||||
class Layer():
|
||||
"""
|
||||
Base class defining a Geneva packet layer.
|
||||
"""
|
||||
protocol = None
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes this layer.
|
||||
"""
|
||||
self.layer = layer
|
||||
# No custom setter, getters, generators, or parsers are needed by default
|
||||
self.setters = {}
|
||||
self.getters = {}
|
||||
self.generators = {}
|
||||
self.parsers = {}
|
||||
|
||||
@classmethod
|
||||
def reset_restrictions(cls):
|
||||
"""
|
||||
Resets field restrictions placed on this layer.
|
||||
"""
|
||||
cls.fields = cls._fields
|
||||
|
||||
def get_next_layer(self):
|
||||
"""
|
||||
Given the current layer returns the next layer beneath us.
|
||||
"""
|
||||
if len(self.layer.layers()) == 1:
|
||||
return None
|
||||
|
||||
return self.layer[1]
|
||||
|
||||
def get_random(self):
|
||||
"""
|
||||
Retreives a random field and value.
|
||||
"""
|
||||
field = random.choice(self.fields)
|
||||
return field, self.get(field)
|
||||
|
||||
def gen_random(self):
|
||||
"""
|
||||
Generates a random field and value.
|
||||
"""
|
||||
assert self.fields, "Layer %s doesn't have any fields" % str(self)
|
||||
field = random.choice(self.fields)
|
||||
return field, self.gen(field)
|
||||
|
||||
@classmethod
|
||||
def name_matches(cls, name):
|
||||
"""
|
||||
Checks if given name matches this layer name.
|
||||
"""
|
||||
return name.upper() == cls.name.upper()
|
||||
|
||||
def get(self, field):
|
||||
"""
|
||||
Retrieves the value from a given field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.getters:
|
||||
return self.getters[field](field)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.getters:
|
||||
return self.getters[base](field)
|
||||
|
||||
return getattr(self.layer, field)
|
||||
|
||||
def set(self, packet, field, value):
|
||||
"""
|
||||
Sets the value for a given field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
base = field.split("-")[0]
|
||||
if field in self.setters:
|
||||
self.setters[field](packet, field, value)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
elif "-" in field and base in self.setters:
|
||||
self.setters[base](packet, field, value)
|
||||
else:
|
||||
setattr(self.layer, field, value)
|
||||
|
||||
# Request the packet be reparsed to confirm the value is stable
|
||||
# XXX Temporarily disabling the reconstitution check due to scapy bug (#2034)
|
||||
#assert bytes(self.protocol(bytes(self.layer))) == bytes(self.layer)
|
||||
|
||||
def gen(self, field):
|
||||
"""
|
||||
Generates a value for this field.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.generators:
|
||||
return self.generators[field](field)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.generators:
|
||||
return self.generators[base](field)
|
||||
|
||||
sample = fuzz(self.protocol())
|
||||
|
||||
new_value = getattr(sample, field)
|
||||
if new_value == None:
|
||||
new_value = 0
|
||||
elif type(new_value) != int:
|
||||
new_value = new_value._fix()
|
||||
|
||||
return new_value
|
||||
|
||||
def parse(self, field, value):
|
||||
"""
|
||||
Parses the given value for a given field. This is useful for fields whose
|
||||
value cannot be represented in a string type easily - it lets us define
|
||||
a common string representation for the strategy, and parse it back into
|
||||
a real value here.
|
||||
"""
|
||||
assert field in self.fields
|
||||
if field in self.parsers:
|
||||
return self.parsers[field](field, value)
|
||||
|
||||
# Dual field accessors are fields that require two pieces of information
|
||||
# to retrieve them (for example, "options-eol"). These are delimited by
|
||||
# a dash "-".
|
||||
base = field.split("-")[0]
|
||||
if "-" in field and base in self.parsers:
|
||||
return self.parsers[base](field, value)
|
||||
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError:
|
||||
parsed = value
|
||||
|
||||
return parsed
|
||||
|
||||
def get_load(self, field):
|
||||
"""
|
||||
Helper method to retrieve load, as scapy doesn't recognize 'load' as
|
||||
a regular field properly.
|
||||
"""
|
||||
try:
|
||||
load = self.layer.payload.load
|
||||
except AttributeError:
|
||||
pass
|
||||
try:
|
||||
load = self.layer.load
|
||||
except AttributeError:
|
||||
return ""
|
||||
|
||||
if not load:
|
||||
return ""
|
||||
|
||||
return urllib.parse.quote(load.decode('utf-8', 'ignore'))
|
||||
|
||||
def set_load(self, packet, field, value):
|
||||
"""
|
||||
Helper method to retrieve load, as scapy doesn't recognize 'load' as
|
||||
a field properly.
|
||||
"""
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].len
|
||||
|
||||
value = urllib.parse.unquote(value)
|
||||
|
||||
value = value.encode('utf-8')
|
||||
dns_payload = b"\x009ib\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x08faceface\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00\x01\x00\x00\x01+\x00\x04\xc7\xbf2I\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00"
|
||||
http_payload = b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
|
||||
|
||||
value = value.replace(b"__DNS_REQUEST__", dns_payload)
|
||||
value = value.replace(b"__HTTP_REQUEST__", http_payload)
|
||||
|
||||
self.layer.payload = Raw(value)
|
||||
|
||||
def gen_load(self, field):
|
||||
"""
|
||||
Helper method to generate a random load, as scapy doesn't recognize 'load'
|
||||
as a field properly.
|
||||
"""
|
||||
load = ''.join([random.choice(string.ascii_lowercase + string.digits) for k in range(10)])
|
||||
return random.choice(["", "__DNS_REQUEST__", "__HTTP_REQUEST__", urllib.parse.quote(load)])
|
|
@ -1,15 +1,19 @@
|
|||
import copy
|
||||
import random
|
||||
|
||||
import actions.layer
|
||||
|
||||
import layers.layer
|
||||
import layers.ip_layer
|
||||
import layers.tcp_layer
|
||||
import layers.udp_layer
|
||||
import layers.dns_layer
|
||||
import layers.dnsqr_layer
|
||||
|
||||
_SUPPORTED_LAYERS = [
|
||||
actions.layer.IPLayer,
|
||||
actions.layer.TCPLayer,
|
||||
actions.layer.UDPLayer,
|
||||
actions.layer.DNSLayer,
|
||||
actions.layer.DNSQRLayer
|
||||
layers.ip_layer.IPLayer,
|
||||
layers.tcp_layer.TCPLayer,
|
||||
layers.udp_layer.UDPLayer,
|
||||
layers.dns_layer.DNSLayer,
|
||||
layers.dnsqr_layer.DNSQRLayer
|
||||
]
|
||||
SUPPORTED_LAYERS = _SUPPORTED_LAYERS
|
||||
|
||||
|
@ -110,10 +114,9 @@ class Packet():
|
|||
"""
|
||||
iter_packet = self.packet
|
||||
while iter_packet:
|
||||
if iter_packet.name.lower() == "raw":
|
||||
return
|
||||
parsed_layer = Packet.parse_layer(iter_packet)
|
||||
if parsed_layer:
|
||||
if parsed_layer.name != "Raw":
|
||||
yield parsed_layer
|
||||
iter_packet = parsed_layer.get_next_layer()
|
||||
else:
|
||||
|
@ -327,6 +330,6 @@ class Packet():
|
|||
"""
|
||||
Performs DNS decompression, if applicable. Returns a new packet.
|
||||
"""
|
||||
self.packet = actions.layer.DNSLayer.dns_decompress(self.packet, logger)
|
||||
self.packet = layers.dns_layer.DNSLayer.dns_decompress(self.packet, logger)
|
||||
self.layers = self.setup_layers()
|
||||
return self
|
|
@ -0,0 +1,10 @@
|
|||
from layers.layer import Layer
|
||||
|
||||
class RawLayer(Layer):
|
||||
"""
|
||||
Defines an interface for the scapy Raw layer.
|
||||
"""
|
||||
name = "Raw"
|
||||
protocol = Raw
|
||||
_fields = []
|
||||
fields = []
|
|
@ -0,0 +1,285 @@
|
|||
import random
|
||||
import binascii
|
||||
from layers.layer import Layer
|
||||
from scapy.all import TCP, fuzz
|
||||
|
||||
class TCPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access TCP header fields.
|
||||
"""
|
||||
name = "TCP"
|
||||
protocol = TCP
|
||||
_fields = [
|
||||
'sport',
|
||||
'dport',
|
||||
'seq',
|
||||
'ack',
|
||||
'dataofs',
|
||||
'reserved',
|
||||
'flags',
|
||||
'window',
|
||||
'chksum',
|
||||
'urgptr',
|
||||
'load',
|
||||
'options-eol',
|
||||
'options-nop',
|
||||
'options-mss',
|
||||
'options-wscale',
|
||||
'options-sackok',
|
||||
'options-sack',
|
||||
'options-timestamp',
|
||||
'options-altchksum',
|
||||
'options-altchksumopt',
|
||||
'options-md5header',
|
||||
'options-uto'
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
options_names = {
|
||||
"eol": 0,
|
||||
"nop": 1,
|
||||
"mss": 2,
|
||||
"wscale": 3,
|
||||
"sackok": 4,
|
||||
"sack": 5,
|
||||
#"echo" : 6,
|
||||
#"echo_reply" : 7,
|
||||
"timestamp": 8,
|
||||
"altchksum": 14,
|
||||
"altchksumopt": 15,
|
||||
"md5header": 19,
|
||||
#"quick_start" : 27,
|
||||
"uto": 28
|
||||
#"authentication": 29,
|
||||
#"experiment": 254
|
||||
}
|
||||
|
||||
# Each entry is Kind: length
|
||||
options_length = {
|
||||
0: 0, # EOL
|
||||
1: 0, # NOP
|
||||
2: 2, # MSS
|
||||
3: 1, # WScale
|
||||
4: 0, # SAckOK
|
||||
5: 0, # SAck
|
||||
6: 4, # Echo
|
||||
7: 4, # Echo Reply
|
||||
8: 8, # Timestamp
|
||||
14: 3, # AltChkSum
|
||||
15: 0, # AltChkSumOpt
|
||||
19: 16, # MD5header Option
|
||||
27: 6, # Quick-Start response
|
||||
28: 2, # User Timeout Option
|
||||
29: 4, # TCP Authentication Option
|
||||
254: 8, # Experiment
|
||||
|
||||
}
|
||||
# Required by scapy
|
||||
scapy_options = {
|
||||
0: "EOL",
|
||||
1: "NOP",
|
||||
2: "MSS",
|
||||
3: "WScale",
|
||||
4: "SAckOK",
|
||||
5: "SAck",
|
||||
8: "Timestamp",
|
||||
14: "AltChkSum",
|
||||
15: "AltChkSumOpt",
|
||||
28: "UTO",
|
||||
# 254:"Experiment" # scapy has two versions of this, so it doesn't work
|
||||
}
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the TCP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
# Special methods to help access fields that cannot be accessed normally
|
||||
self.getters = {
|
||||
'load' : self.get_load,
|
||||
'options' : self.get_options
|
||||
}
|
||||
self.setters = {
|
||||
'load' : self.set_load,
|
||||
'options' : self.set_options
|
||||
}
|
||||
# Special methods to help generate fields that cannot be generated normally
|
||||
self.generators = {
|
||||
'load' : self.gen_load,
|
||||
'dataofs' : self.gen_dataofs,
|
||||
'flags' : self.gen_flags,
|
||||
'chksum' : self.gen_chksum,
|
||||
'options' : self.gen_options,
|
||||
'window' : self.gen_window
|
||||
}
|
||||
|
||||
def gen_window(self, field):
|
||||
"""
|
||||
Generates a window size.
|
||||
"""
|
||||
return random.choice(range(10, 200, 10))
|
||||
|
||||
def gen_chksum(self, field):
|
||||
"""
|
||||
Generates a checksum.
|
||||
"""
|
||||
return random.randint(1, 65535)
|
||||
|
||||
def gen_dataofs(self, field):
|
||||
"""
|
||||
Generates a valid value for the data offset field.
|
||||
"""
|
||||
# Dataofs is a 4 bit header, so a max of 15
|
||||
return random.randint(1, 15)
|
||||
|
||||
def gen_flags(self, field):
|
||||
"""
|
||||
Generates a random set of flags. 50% of the time it picks randomly from
|
||||
a list of real flags, otherwise it returns fuzzed flags.
|
||||
"""
|
||||
if random.random() < 0.5:
|
||||
return random.choice(['S', 'A', 'SA', 'PA', 'FA', 'R', 'P', 'F', 'RA', ''])
|
||||
else:
|
||||
sample = fuzz(self.protocol())
|
||||
# Since scapy lazily evaluates fuzzing, we first must set a
|
||||
# legitimate value for scapy to evaluate what combination of flags it is
|
||||
sample.flags = sample.flags
|
||||
return str(sample.flags)
|
||||
|
||||
def get_options(self, field):
|
||||
"""
|
||||
Helper method to retrieve options.
|
||||
"""
|
||||
base, req_option = field.split("-")
|
||||
assert base == "options", "get_options can only be used to fetch options."
|
||||
option_type = self.option_str_to_int(req_option)
|
||||
i = 0
|
||||
# First, check if the option is already present in the packet
|
||||
for option in self.layer.options:
|
||||
# Scapy may try to be helpful and return the string of the option
|
||||
next_option = self.option_str_to_int(option[0])
|
||||
if option_type == next_option:
|
||||
_name, value = self.layer.options[i]
|
||||
# Some options (timestamp, checksums, nop) store their value in a
|
||||
# tuple.
|
||||
if isinstance(value, tuple):
|
||||
# Scapy returns values in any of these types
|
||||
if value in [None, b'', ()]:
|
||||
return ''
|
||||
value = value[0]
|
||||
if value in [None, b'', ()]:
|
||||
return ''
|
||||
if req_option == "md5header":
|
||||
return binascii.hexlify(value).decode("utf-8")
|
||||
|
||||
return value
|
||||
i += 1
|
||||
return ''
|
||||
|
||||
def set_options(self, packet, field, value):
|
||||
"""
|
||||
Helper method to set options.
|
||||
"""
|
||||
base, option = field.split("-")
|
||||
assert base == "options", "Must use an options field with set_options"
|
||||
|
||||
option_type = self.option_str_to_int(option)
|
||||
if type(value) == str:
|
||||
# Prepare the value for storage in the packet
|
||||
value = binascii.unhexlify(value)
|
||||
|
||||
# Scapy requires these options to be a tuple - since evaling this
|
||||
# is not yet supported, for now, SAck will always be an empty tuple
|
||||
if option in ["sack"]:
|
||||
value = ()
|
||||
# These options must be set as integers - if they didn't exist, they can
|
||||
# be added like this
|
||||
if option in ["timestamp", "mss", "wscale", "altchksum", "uto"] and not value:
|
||||
value = 0
|
||||
i = 0
|
||||
# First, check if the option is already present in the packet
|
||||
for option in self.layer.options:
|
||||
# Scapy may try to be helpful and return the string of the option
|
||||
next_option = self.option_str_to_int(option[0])
|
||||
|
||||
if option_type == next_option:
|
||||
packet["TCP"].options[i] = self.format_option(option_type, value)
|
||||
break
|
||||
i += 1
|
||||
# If we didn't break, the option doesn't exist in the packet currently.
|
||||
else:
|
||||
old_options_array = packet["TCP"].options
|
||||
old_options_array.append(self.format_option(option_type, value))
|
||||
packet["TCP"].options = old_options_array
|
||||
|
||||
# Let scapy recalculate the required values
|
||||
del self.layer.chksum
|
||||
del self.layer.dataofs
|
||||
if packet.haslayer("IP"):
|
||||
del packet["IP"].chksum
|
||||
del packet["IP"].len
|
||||
return True
|
||||
|
||||
def gen_options(self, field):
|
||||
"""
|
||||
Helper method to set options.
|
||||
"""
|
||||
_, option = field.split("-")
|
||||
option_num = self.options_names[option]
|
||||
length = self.options_length[option_num]
|
||||
|
||||
data = b''
|
||||
if length > 0:
|
||||
data = os.urandom(length)
|
||||
data = binascii.hexlify(data).decode()
|
||||
# MSS must be a 2-byte int
|
||||
if option_num == 2:
|
||||
data = random.randint(0, 65535)
|
||||
# WScale must be a 1-byte int
|
||||
elif option_num == 3:
|
||||
data = random.randint(0, 255)
|
||||
# Timestamp must be an int
|
||||
elif option_num == 8:
|
||||
data = random.randint(0, 4294967294)
|
||||
elif option_num == 14:
|
||||
data = random.randint(0, 255)
|
||||
elif option_num == 28:
|
||||
data = random.randint(0, 255)
|
||||
|
||||
return data
|
||||
|
||||
def option_str_to_int(self, option):
|
||||
"""
|
||||
Takes a string representation of an option and returns the option integer code.
|
||||
"""
|
||||
if type(option) == int:
|
||||
return option
|
||||
|
||||
assert "-" not in option, "Must be given specific option: %s." % option
|
||||
|
||||
for val in self.scapy_options:
|
||||
if self.scapy_options[val].lower() == option.lower():
|
||||
return val
|
||||
|
||||
if " " in option:
|
||||
option = option.replace(" ", "_").lower()
|
||||
|
||||
if option.lower() in self.options_names:
|
||||
return self.options_names[option.lower()]
|
||||
|
||||
def format_option(self, options_int, value):
|
||||
"""
|
||||
Formats the options so they will work with scapy.
|
||||
"""
|
||||
# NOPs
|
||||
if options_int == 1:
|
||||
return (self.scapy_options[options_int], ())
|
||||
elif options_int in [5]:
|
||||
return (self.scapy_options[options_int], value)
|
||||
# Timestamp
|
||||
elif options_int in [8, 14]:
|
||||
return (self.scapy_options[options_int], (value, 0))
|
||||
elif options_int in self.scapy_options:
|
||||
return (self.scapy_options[options_int], value)
|
||||
else:
|
||||
return (options_int, value)
|
|
@ -0,0 +1,32 @@
|
|||
from layers.layer import Layer
|
||||
from scapy.all import UDP
|
||||
|
||||
class UDPLayer(Layer):
|
||||
"""
|
||||
Defines an interface to access UDP header fields.
|
||||
"""
|
||||
name = "UDP"
|
||||
protocol = UDP
|
||||
_fields = [
|
||||
"sport",
|
||||
"dport",
|
||||
"chksum",
|
||||
"len",
|
||||
"load"
|
||||
]
|
||||
fields = _fields
|
||||
|
||||
def __init__(self, layer):
|
||||
"""
|
||||
Initializes the UDP layer.
|
||||
"""
|
||||
Layer.__init__(self, layer)
|
||||
self.getters = {
|
||||
'load' : self.get_load,
|
||||
}
|
||||
self.setters = {
|
||||
'load' : self.set_load,
|
||||
}
|
||||
self.generators = {
|
||||
'load' : self.gen_load,
|
||||
}
|
|
@ -0,0 +1,480 @@
|
|||
# The following strategies have been learned as successful against the Great Firewall.
|
||||
WORKING_STRATEGIES = [
|
||||
{
|
||||
"strategy" : "\/",
|
||||
"success_rate" : .03,
|
||||
"description" : "No strategy",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
|
||||
# TCB Desync - High DataOfs
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:replace:25776},),)-",
|
||||
"success_rate" : .98,
|
||||
"description" : "TCP Desync - Increment Dataofs - Corrupt Chksum",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-",
|
||||
"success_rate" : .98,
|
||||
"description" : "TCP Desync - Increment Dataofs - Small TTL",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:flags:replace:FRAPUN},),)-",
|
||||
"success_rate" : .26,
|
||||
"description" : "TCP Desync - Increment Dataofs - Invalid Flags",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-",
|
||||
"success_rate" : .94,
|
||||
"description" : "TCP Desync - Increment Dataofs - Corrupt ACK",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-",
|
||||
"success_rate" : .98,
|
||||
"description" : "TCP Desync - Increment Dataofs - Corrupt WScale",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# TCB Desync - Load corruption
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-",
|
||||
"success_rate" : .98,
|
||||
"description" : "TCP Desync - Invalid Payload - Corrupt Chksum",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8}(duplicate(fragment{tcp:-1:False},),),),)-",
|
||||
"success_rate" : .98,
|
||||
"description" : "TCP Desync - Invalid Payload - Small TTL",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt}(duplicate(fragment{tcp:-1:False},),),),)-",
|
||||
"success_rate" : .93,
|
||||
"description" : "TCP Desync - Invalid Payload - Corrupt ACK",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
|
||||
# TCB Teardown (with RST)
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-",
|
||||
"success_rate" : .95,
|
||||
"description" : "TCB Teardown - with RST - Corrupt Chksum, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-",
|
||||
"success_rate" : .51,
|
||||
"description" : "TCB Teardown - with RST - Corrupt Chksum, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-",
|
||||
"success_rate" : .87,
|
||||
"description" : "TCB Teardown - with RST - Small TTL, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:9},),)-",
|
||||
"success_rate" : .52,
|
||||
"description" : "TCB Teardown - with RST - Small TTL, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-",
|
||||
"success_rate" : .86,
|
||||
"description" : "TCB Teardown - with RST - Invalid md5Header, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:RA},),)-",
|
||||
"success_rate" : .44,
|
||||
"description" : "TCB Teardown - with RST - Invalid md5Header, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# Teardown with RST/ACK
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:replace:27925},))-",
|
||||
"success_rate" : .90,
|
||||
"description" : "TCB Teardown - with RST/ACK - Corrupt Chksum, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:replace:27925},),)-",
|
||||
"success_rate" : .66,
|
||||
"description" : "TCB Teardown - with RST/ACK - Corrupt Chksum, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-",
|
||||
"success_rate" : .94,
|
||||
"description" : "TCB Teardown - with RST/ACK - Small TTL, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},),)-",
|
||||
"success_rate" : .57,
|
||||
"description" : "TCB Teardown - with RST/ACK - Small TTL, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-",
|
||||
"success_rate" : .94,
|
||||
"description" : "TCB Teardown - with RST/ACK - Invalid md5Header, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},),)-",
|
||||
"success_rate" : .48,
|
||||
"description" : "TCB Teardown - with RST/ACK - Invalid md5Header, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{TCP:ack:corrupt},),)-",
|
||||
"success_rate" : .43,
|
||||
"description" : "TCB Teardown - with RST/ACK - Corrupt ACK, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:ack:corrupt},))-",
|
||||
"success_rate" : .31,
|
||||
"description" : "TCB Teardown - with RST/ACK - Corrupt ACK, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# TCB Teardown w/ Invalid Flags
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))- ",
|
||||
"success_rate" : .89,
|
||||
"description" : "TCB Teardown - Invalid Flags - Corrupt Chksum, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},),)-",
|
||||
"success_rate" : .48,
|
||||
"description" : "TCB Teardown - Invalid Flags - Corrupt Chksum, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-",
|
||||
"success_rate" : .96,
|
||||
"description" : "TCB Teardown - Invalid Flags - Small TTL, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{IP:ttl:replace:10},),)-",
|
||||
"success_rate" : .56,
|
||||
"description" : "TCB Teardown - Invalid Flags - Small TTL, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-",
|
||||
"success_rate" : .94,
|
||||
"description" : "TCB Teardown - Invalid Flags - Invalid md5Header, High Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:options-md5header:corrupt},),)-",
|
||||
"success_rate" : .55,
|
||||
"description" : "TCB Teardown - Invalid Flags - Invalid md5Header, Low Percentage",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-tamper{TCP:load:replace:a5z2}(tamper{TCP:flags:replace:CN},)-",
|
||||
"success_rate" : .00,
|
||||
"description" : "TCB Teardown - Invalid Flags - Invalid Payload",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# Segmentation
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:8:False}-|[TCP:flags:A]-tamper{TCP:seq:replace:2258679050}-",
|
||||
"success_rate" : .92,
|
||||
"description" : "Segmentation - With ACK - Offsets, subvariant 1",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:replace:2258679050}(tamper{TCP:flags:replace:PA},)-",
|
||||
"success_rate" : .95,
|
||||
"description" : "Segmentation - With ACK - Offsets, subvariant 2",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-",
|
||||
"success_rate" : .98,
|
||||
"description" : "Segmentation - Reassembly - Offsets, subvariant 1",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:4:True}(,fragment{tcp:19:True})-",
|
||||
"success_rate" : .96,
|
||||
"description" : "Segmentation - Reassembly - Offsets, subvariant 2",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# Hybrid Strategies
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:68},),)-| \/",
|
||||
"success_rate" : .53,
|
||||
"description" : "Hybrid Strategy - HTTP - Fragment and set Fin flag",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:68},),)-| \/",
|
||||
"success_rate" : 1,
|
||||
"description" : "Hybrid Strategy - DNS Over TCP - Fragment and set Fin flag",
|
||||
"test_type" : "dns_tcp",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# Server-side strategies
|
||||
{
|
||||
"strategy" : "[TCP:flags:SA]-duplicate(tamper{TCP:flags:replace:FREC}(duplicate,),tamper{TCP:flags:replace:FSEC})-| \/",
|
||||
"success_rate" : .43,
|
||||
"description" : "Server-Side - Simultaneous Open",
|
||||
"test_type" : "http",
|
||||
"server_side" : True,
|
||||
"country" : "china"
|
||||
},
|
||||
# DNS over UDP strategies
|
||||
{
|
||||
"strategy" : "\/ [UDP:sport:53:2]-drop-",
|
||||
"success_rate" : .96,
|
||||
"description" : "DNS - Drop first two bad responses",
|
||||
"test_type" : "dns",
|
||||
"server_side" : False,
|
||||
"country" : "china"
|
||||
},
|
||||
# India HTTP strategies
|
||||
{
|
||||
"strategy" : "[TCP:options-altchksum:]-tamper{TCP:options-uto:corrupt}(duplicate,)-| \/",
|
||||
"success_rate" : 1,
|
||||
"description" : "Invalid Options",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:options-mss:]-tamper{TCP:options-md5header:corrupt}-| \/",
|
||||
"success_rate" : 1,
|
||||
"description" : "Invalid Options",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:9},)-| \/",
|
||||
"success_rate" : 1,
|
||||
"description" : "Increasing dataofs",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "IP length",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:-1:True}-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "Segmentation",
|
||||
"test_type" : "http",
|
||||
"server_side" : False,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:SA]-tamper{TCP:window:replace:98}-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "Server side",
|
||||
"test_type" : "http",
|
||||
"server_side" : True,
|
||||
"country" : "india"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-fragment{tcp:-1:True}-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "Segmentation",
|
||||
"test_type" : "kazakhstan_injected_https",
|
||||
"server_side" : False,
|
||||
"country" : "kazakhstan"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:PA]-duplicate(tamper{IP:len:replace:78},)-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "Segmentation Exploit - Small IP length",
|
||||
"test_type" : "kazakhstan_injected_https",
|
||||
"server_side" : False,
|
||||
"country" : "kazakhstan"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "Desync: load on second SYN",
|
||||
"test_type" : "kazakhstan_injected_https",
|
||||
"server_side" : False,
|
||||
"country" : "kazakhstan"
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|",
|
||||
"success_rate" : 1,
|
||||
"description" : "TCB Turnaround",
|
||||
"test_type" : "kazakhstan_injected_https",
|
||||
"server_side" : False,
|
||||
"country" : "kazakhstan"
|
||||
},
|
||||
]
|
||||
|
||||
# Strategies that evade the lab censors
|
||||
LAB_STRATEGIES = [
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-tamper{TCP:flags:replace:F}-| \/",
|
||||
"censors" : ["censor6", "censor7", "censor8"],
|
||||
"description" : "Interrupts the 3-way handshake with a FIN. The server ignores the FIN, as \
|
||||
the 3-way handshake has not been done yet, so there is no connection to teardown. \
|
||||
Since the client's very next packet is a PSH-ACK, the ACK in this packet serves to \
|
||||
complete the 3-way handshake, but the censor tears down it's TCB at the FIN."
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:seq:corrupt}(tamper{TCP:flags:replace:R},),)-| \/",
|
||||
"censors" : ["censor6", "censor7"],
|
||||
"description" : "Tears down a TCB right after a 3-way handshake by injecting a RST packet."
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/",
|
||||
"censors" : ["censor6", "censor7", "censor8", "censor8b"],
|
||||
"description" : "Creates a RST injection packet to tear down a TCB and corrupts the checksum, exploiting a censor \
|
||||
that does not validate packet checksums."
|
||||
},
|
||||
{
|
||||
"strategy" : "\/ [TCP:dataofs:5]-drop-|",
|
||||
"censors" : ["censor2"],
|
||||
"description" : "RST Packets (at least those constructed with scapy) have a data offset of 5. This \
|
||||
beats a censor that only sends RSTs to the client by dropping the RST packets."
|
||||
},
|
||||
{
|
||||
"strategy" : "\/ [TCP:flags:R]-drop-|",
|
||||
"censors" : ["censor2"],
|
||||
"description" : "Beats a censor that only sends RSTs to the client by dropping the RST packets."
|
||||
},
|
||||
{
|
||||
"strategy" : "\/ [TCP:window:8192]-drop-|",
|
||||
"censors" : ["censor2"],
|
||||
"description" : "RST Packets (at least those constructed with scapy) have a default window size of 8192. This \
|
||||
beats a censor that only sends RSTs to the client by dropping the RST packets."
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:replace:14239},),duplicate(tamper{TCP:flags:replace:S}(tamper{TCP:chksum:replace:14239},),))-| \/",
|
||||
"censors" : ["censor1", "censor2", "censor3", "censor5", "censor6", "censor7", "censor8", "censor8b", "censor9"],
|
||||
"description" : "Triggers a RST packet and SYN packet to be sent immediately after the 3-way handshake \
|
||||
finishes. The server ignores the RST, as the chksum is corrupted, and ignores the SYN, \
|
||||
as a connection is already up. The censor sees the RST and enters the resynchronization \
|
||||
state, and the immediate follow-up SYN packet causes the TCB to be deschronized from the \
|
||||
real connection."
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:replace:15239},),duplicate(tamper{TCP:flags:replace:S}(tamper{TCP:seq:corrupt}(tamper{TCP:chksum:corrupt},),),))-| \/",
|
||||
|
||||
"censors" : ["censor1", "censor2", "censor3", "censor5", "censor6", "censor7", "censor8", "censor8b", "censor9"],
|
||||
"description" : "Triggers a RST packet and SYN packet to be sent immediately after the 3-way handshake \
|
||||
finishes. The server ignores the RST, as the chksum is corrupted, and ignores the SYN, \
|
||||
as a connection is already up. The censor sees the RST and enters the resynchronization \
|
||||
state, and the immediate follow-up SYN packet with a new seq causes the TCB to be deschronized from the \
|
||||
real connection."
|
||||
},
|
||||
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-tamper{TCP:dataofs:replace:0}-| \/",
|
||||
"censors" : ["censor1", "censor2", "censor3", "censor5", "censor9", "censor10"],
|
||||
"description" : "The dataofs field in the TCP header tells applications where the payload of the packet \
|
||||
starts. By replacing the dataofs to 0 on a packet without a payload (ACK), it makes the \
|
||||
TCP header look like data. Servers ignore this, but a censor that is trying to keep a TCB \
|
||||
synchronized will be desynchronized from the connection when it gets a payload of an incorrect \
|
||||
length."
|
||||
|
||||
},
|
||||
{
|
||||
"strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:dataofs:replace:0},)-| \/",
|
||||
"censors" : ["censor1", "censor2", "censor3", "censor5", "censor9", "censor10"],
|
||||
"description" : "The dataofs field in the TCP header tells applications where the payload of the packet \
|
||||
starts. By replacing the dataofs to 0 on a packet without a payload (ACK), it makes the \
|
||||
TCP header look like data. Servers ignore this, but a censor that is trying to keep a TCB \
|
||||
synchronized will be desynchronized from the connection when it gets a payload of an incorrect \
|
||||
length. This strategy is functionally equivalent to the above strategy, but also preserves the \
|
||||
original packet."
|
||||
|
||||
},
|
||||
]
|
|
@ -0,0 +1,171 @@
|
|||
"""
|
||||
Client
|
||||
|
||||
Run by the evaluator, sends data to discard server.
|
||||
|
||||
Not usually used for training because in Python it is difficult to distinguish between a successful
|
||||
strategy and an unsuccessful strategy. This is because in the discard protocol (in a good case),
|
||||
the client will send data and the server will throw it away (but ACK it). In a bad case (such as a
|
||||
failing strategy that breaks the TCP connection), the client sends the data but it does not reach
|
||||
the server (so it is not ACKed). However, in Python, it is non trivial to distinguish these two
|
||||
cases, as neither send() nor sendall() will raise a timeout or check if the data that is sent is
|
||||
ACKed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.request
|
||||
|
||||
import requests
|
||||
|
||||
socket.setdefaulttimeout(10)
|
||||
|
||||
import actions.utils
|
||||
|
||||
from plugins.plugin_client import ClientPlugin
|
||||
|
||||
import signal
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
# Sets up timeout signal because currently, sendall does not give a timeout which is making the
|
||||
# client believe that the connection has not been torn down.
|
||||
class TimeoutError(Exception):
|
||||
"""
|
||||
Houses a TimeoutError so we can cut off sendall.
|
||||
"""
|
||||
pass
|
||||
|
||||
def handle_timeout(signum, frame):
|
||||
import errno
|
||||
raise TimeoutError(os.strerror(errno.ETIME))
|
||||
|
||||
|
||||
class DiscardClient(ClientPlugin):
|
||||
"""
|
||||
Defines the Discard client.
|
||||
"""
|
||||
name = "discard"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the discard client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines required args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='Discard Client')
|
||||
|
||||
# If we know whether the censor we are training against injects content, we can optimize the plugin's behavior.
|
||||
# Censors that inject content will give us a very clear signal - after we send content, we can use `recv()` to get
|
||||
# the response from the censor. Since this is the discard protocol, if we ever receive data, this is from the censor.
|
||||
parser.add_argument('--injection-censor', action='store_true', help="whether this censor injects content or sends RSTs to censor")
|
||||
parser.add_argument('--server', action='store', help="server to connect to")
|
||||
# Makes it easier to craft fake HTTP requests to trip censorship
|
||||
parser.add_argument('--http-request', action='store', help="send an HTTP get request with the given hostname to the discard server")
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to make a forbidden GET request to the server.
|
||||
"""
|
||||
fitness = 0
|
||||
port = int(args["port"])
|
||||
server = args["server"]
|
||||
bad_word = args["bad_word"]
|
||||
msg = bad_word
|
||||
injection_censor = args.get("injection_censor")
|
||||
if args.get("http_request"):
|
||||
msg = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % args.get("http_request")
|
||||
try:
|
||||
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
client.connect((server, port))
|
||||
for idx in range(0, 5):
|
||||
if type(msg) == str:
|
||||
msg = msg.encode()
|
||||
|
||||
# Set a 10 second timeout on the socket. Timeouts do not interrupt send() or sendall().
|
||||
client.settimeout(10)
|
||||
|
||||
# Manually create a 5 second timeout
|
||||
timeout = 5
|
||||
|
||||
# Setup the timeout as a signal alarm
|
||||
signal.signal(signal.SIGALRM, handle_timeout)
|
||||
signal.alarm(timeout)
|
||||
|
||||
reached_timeout = False
|
||||
try:
|
||||
client.send(msg)
|
||||
# Give the alarm time to realize it must go off
|
||||
time.sleep(1)
|
||||
except TimeoutError:
|
||||
logger.debug("sendall() timed out")
|
||||
fitness -= 100
|
||||
reached_timeout = True
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
|
||||
# If the censor injects content, checks to make sure nothing is sent back from the server.
|
||||
# If the recv times out, then the procedure was successful. If injected content is sent back,
|
||||
# censorship has occurred.
|
||||
if injection_censor:
|
||||
try:
|
||||
server_data = client.recv(1024)
|
||||
logger.debug("Data received: %s", server_data.decode('utf-8', 'ignore'))
|
||||
fitness -= 90
|
||||
except socket.timeout:
|
||||
fitness += 100
|
||||
logger.debug("No data received from a censor.")
|
||||
# If the censor is not an injection censor and the connection is not teared down, the strategy
|
||||
# is successful.
|
||||
else:
|
||||
if idx != 0 and not reached_timeout:
|
||||
fitness += 90
|
||||
client.close()
|
||||
# If the fitness is 0, the strategy did something to corrupt/interfere with the socket
|
||||
# sending/receiving, usually by just artificially closing the connection. This behavior
|
||||
# should not be rewarded with a higher fitness
|
||||
if fitness == 0:
|
||||
fitness -= 100
|
||||
|
||||
except socket.error as exc:
|
||||
# If the censor we're running against tears down connects via RSTs, we can punish RSTs as
|
||||
# if the strategy did not harm the underlying connection. However, if the censor only injects
|
||||
# traffic, not resets, we should punish RSTs harshly, as the strategy likely caused it.
|
||||
|
||||
if exc.errno == 104:
|
||||
if injection_censor:
|
||||
fitness -= 110
|
||||
else:
|
||||
fitness -= 100
|
||||
logger.debug("Client: Connection RST.")
|
||||
else:
|
||||
fitness -= 100
|
||||
logger.exception("Socket error caught in client discard test.")
|
||||
except Exception:
|
||||
logger.exception("Exception caught in client discard test.")
|
||||
fitness = -120
|
||||
finally:
|
||||
logger.debug("Client finished discard test.")
|
||||
signal.alarm(0)
|
||||
return fitness * 4
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
import argparse
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
from plugins.plugin_server import ServerPlugin
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class DiscardServer(ServerPlugin):
|
||||
"""
|
||||
Defines the Discard client.
|
||||
"""
|
||||
name = "discard"
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the client.
|
||||
"""
|
||||
ServerPlugin.__init__(self)
|
||||
self.args = args
|
||||
if args:
|
||||
self.port = args["port"]
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines arguments for this plugin
|
||||
"""
|
||||
super_args = ServerPlugin.get_args(command)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Discard Server')
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger):
|
||||
"""
|
||||
Initializes the Discard server.
|
||||
"""
|
||||
logger.debug("Discard server initializing")
|
||||
try:
|
||||
port = int(args["port"])
|
||||
control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
# Allow socket re-use
|
||||
control_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server_address = ('0.0.0.0', port)
|
||||
logger.debug("Binding to server address 0.0.0.0:%d" % port)
|
||||
control_socket.bind(server_address)
|
||||
control_socket.settimeout(5)
|
||||
control_socket.listen(1)
|
||||
except:
|
||||
logger.exception("Caught exception in discard run")
|
||||
return
|
||||
|
||||
try:
|
||||
connection, client_address = self.get_request(control_socket)
|
||||
if not connection:
|
||||
logger.error("Failed to get connection")
|
||||
return
|
||||
for i in range(0, 5):
|
||||
data = connection.recv(1024)
|
||||
|
||||
connection.close()
|
||||
except socket.error as e:
|
||||
if e.errno == 104:
|
||||
logger.debug("Server: Connection RST.")
|
||||
else:
|
||||
logger.debug("Server: Client quit.")
|
||||
except socket.ConnectionResetError:
|
||||
logger.debug("Server: Connection RST.")
|
||||
except Exception:
|
||||
logger.exception("Failed during server communication.")
|
||||
finally:
|
||||
logger.debug("Server exiting")
|
||||
|
||||
def get_request(self, control_socket):
|
||||
"""
|
||||
Get a request from the socket.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
sock, addr = control_socket.accept()
|
||||
sock.settimeout(5)
|
||||
return (sock, addr)
|
||||
except socket.timeout:
|
||||
pass
|
||||
return (None, None)
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stops this server.
|
||||
"""
|
||||
ServerPlugin.stop(self)
|
|
@ -0,0 +1,122 @@
|
|||
"""
|
||||
Client
|
||||
|
||||
Run by the evaluator, tries to make a GET request to a given server
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.request
|
||||
|
||||
import dns.resolver
|
||||
import requests
|
||||
|
||||
import actions.utils
|
||||
from plugins.plugin_client import ClientPlugin
|
||||
|
||||
|
||||
class DNSClient(ClientPlugin):
|
||||
"""
|
||||
Defines the DNS client.
|
||||
"""
|
||||
name = "dns"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the DNS client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines required args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='DNS Client')
|
||||
|
||||
parser.add_argument('--use-tcp', action='store_true', help='leverage TCP for this plugin')
|
||||
parser.add_argument('--dns-server', action='store', default="8.8.8.8", help='domain server to connect to')
|
||||
parser.add_argument('--query', action='store', default="facebook.com", help='censored domain to query')
|
||||
parser.add_argument('--timeout', action='store', default="3", type=int, help='how long in seconds the client should wait for a response')
|
||||
parser.add_argument('--port', action='store', default="53", type=int, help='port the DNS server is running on (must be 53)')
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to make a forbidden DNS query.
|
||||
"""
|
||||
fitness = 0
|
||||
to_lookup = args.get("query", "facebook.com")
|
||||
dns_server = args.get("dns_server", "8.8.8.8")
|
||||
use_tcp = args.get("use_tcp", False)
|
||||
assert dns_server, "Cannot launch DNS test with no DNS server"
|
||||
assert to_lookup, "Cannot launch DNS test with no server to query"
|
||||
fitness = -1000
|
||||
try:
|
||||
fitness = self.dns_test(to_lookup, dns_server, args["output_directory"], args["environment_id"], logger, timeout=args.get("timeout", 3), use_tcp=use_tcp)
|
||||
except Exception:
|
||||
logger.exception("Exception caught in DNS test to resolver %s.", dns_server)
|
||||
fitness += -100
|
||||
|
||||
# When performing a DNS test, a timeout is indistinguishable from
|
||||
# a reset, which means we can't tell if the strategy broke the packet
|
||||
# stream, or if the censor caught us. Strategies that break the stream
|
||||
# should be punished more harshly, so raise the fitness slightly
|
||||
# if the engine detected censorship for failed DNS tests.
|
||||
if use_tcp and engine and engine.censorship_detected and fitness < 0:
|
||||
fitness += 10
|
||||
return fitness * 4
|
||||
|
||||
def dns_test(self, to_lookup, dns_server, output_dir, environment_id, logger, timeout=3, use_tcp=False):
|
||||
"""
|
||||
Makes a DNS query to a given censored domain.
|
||||
"""
|
||||
# Make the path an absolute path
|
||||
if not output_dir.startswith("/"):
|
||||
output_dir = os.path.join(actions.utils.PROJECT_ROOT, output_dir)
|
||||
|
||||
resolver = dns.resolver.Resolver()
|
||||
protocol = "UDP"
|
||||
if use_tcp:
|
||||
protocol = "TCP"
|
||||
|
||||
logger.debug("Querying %s to DNS server %s over %s" % (to_lookup, dns_server, protocol))
|
||||
resolver.nameservers = [dns_server]
|
||||
# Setup the timeout and lifetime for this resolver
|
||||
resolver.timeout = timeout
|
||||
resolver.lifetime = 3
|
||||
|
||||
try:
|
||||
answer = resolver.query(to_lookup, "A", tcp=use_tcp)[0]
|
||||
logger.debug("Got IP address: %s" % answer)
|
||||
# At this point, we've been given an IP address by the DNS resolver, but we don't
|
||||
# yet know if this IP address is a bogus injected response, or legitimate. Further,
|
||||
# because we are likely running this code from within a censored regime which might
|
||||
# employ secondary censorship at the IP level, we cannot check if this IP is legit
|
||||
# here. Instead, we write it out to a file for the evaluator to extract and check for us.
|
||||
with open(os.path.join(output_dir, "flags", environment_id)+".dnsresult", "w") as dnsfile:
|
||||
dnsfile.write(str(answer))
|
||||
# For now, set fitness to a positive metric, though the evaluator will lower it if
|
||||
# the IP address we were given was bogus.
|
||||
fitness = 100
|
||||
except dns.exception.Timeout:
|
||||
logger.error("DNS query timed out.")
|
||||
fitness = -100
|
||||
except dns.resolver.NoNameservers:
|
||||
logger.error("DNS server failed to respond")
|
||||
fitness = -100
|
||||
|
||||
return fitness
|
|
@ -0,0 +1,143 @@
|
|||
"""
|
||||
DNS Plugin driver
|
||||
|
||||
Overrides the default evaluator plugin handling so we can check for legit IPs for UDP tests.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import calendar
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
import urllib.request
|
||||
|
||||
import requests
|
||||
|
||||
socket.setdefaulttimeout(1)
|
||||
|
||||
import actions.utils
|
||||
|
||||
from plugins.plugin import Plugin
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH))
|
||||
|
||||
|
||||
class DNSPluginRunner(Plugin):
|
||||
"""
|
||||
Defines the DNS plugin runner.
|
||||
"""
|
||||
name = "dns"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Marks this plugin as enabled
|
||||
"""
|
||||
self.enabled = True
|
||||
|
||||
def check_legit_ip(self, ip, logger, domain="facebook"):
|
||||
"""
|
||||
Helper method to check if the given IP address is serving web content.
|
||||
"""
|
||||
url = "http://%s" % ip
|
||||
logger.debug("Checking %s if returned legitimate %s" % (url, domain))
|
||||
try:
|
||||
res = requests.get(url, allow_redirects=False, timeout=3)
|
||||
if res.status_code == 400:
|
||||
res.raise_for_status()
|
||||
# If we got a 301 redirect, the res.text will be empty, but facebook will show up in
|
||||
# the headers
|
||||
for header in res.headers:
|
||||
if domain in res.headers[header]:
|
||||
return True
|
||||
# Otherwise, check the res.text
|
||||
return domain in res.text
|
||||
except Exception as exc:
|
||||
logger.debug("Exception caught in checking DNS result %s: %s", url, exc)
|
||||
return False
|
||||
|
||||
def start(self, args, evaluator, environment, ind, logger):
|
||||
"""
|
||||
Runs the plugins
|
||||
"""
|
||||
# Start the server
|
||||
port = args.get("port", 53)
|
||||
use_tcp = evaluator.client_args.get("use_tcp", False)
|
||||
|
||||
if port != 53:
|
||||
logger.warning("Warning: Given port %s, but GFW only censors on port 53.", str(port))
|
||||
|
||||
# Disable wait for server - it checks based on binding to a TCP port
|
||||
evaluator.server_args.update({"no_wait_for_server" : True})
|
||||
|
||||
# If we're given a server to start, start it now
|
||||
if evaluator.server_cls and not args.get("external_server"):
|
||||
# If a test using TCP has been requested, switch the server to that mode
|
||||
if use_tcp:
|
||||
evaluator.server_args.update({"listener": "socket_TCP"})
|
||||
server = evaluator.start_server(evaluator.server_args, environment, logger)
|
||||
evaluator.client_args.update({"dns_server": evaluator.args["server"]})
|
||||
|
||||
fitness = evaluator.run_client(evaluator.client_args, environment, logger)
|
||||
|
||||
if evaluator.server_cls and not evaluator.args["external_server"]:
|
||||
evaluator.stop_server(environment, server)
|
||||
|
||||
evaluator.read_fitness(ind)
|
||||
|
||||
# If the engine ran on the server side, ask that it punish fitness
|
||||
if evaluator.args["server_side"]:
|
||||
ind.fitness = server.punish_fitness(ind.fitness, logger)
|
||||
# When performing a DNS test, a timeout is indistinguishable from
|
||||
# a reset, which means we can't tell if the strategy broke the packet
|
||||
# stream, or if the censor caught us. Strategies that break the stream
|
||||
# should be punished more harshly, so raise the fitness slightly
|
||||
# if the engine detected censorship for failed DNS tests.
|
||||
if use_tcp and server.engine and server.engine.censorship_detected and ind.fitness < 0:
|
||||
logger.debug("Censorship detected - adjusting positively for not killing stream")
|
||||
ind.fitness += 40
|
||||
|
||||
output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory"))
|
||||
fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness"
|
||||
with open(fitpath, "w") as fitfile:
|
||||
fitfile.write(str(ind.fitness))
|
||||
|
||||
if evaluator.args["external_client"]:
|
||||
command = 'cat %s/%s/%s/%s.dnsresult' % (environment["worker"]["geneva_path"], evaluator.args["output_directory"], actions.utils.FLAGFOLDER, environment["id"])
|
||||
dns_result, error_lines = evaluator.remote_exec_cmd(environment["remote"], command, logger)
|
||||
if not dns_result:
|
||||
logger.debug("Failed to get DNS result.")
|
||||
else:
|
||||
result = dns_result[0]
|
||||
logger.debug("Got result: %s" % result)
|
||||
# If the IP we got back was bad, we must fail the strategy
|
||||
if not self.check_legit_ip(result, logger, domain="facebook"):
|
||||
ind.fitness = -360
|
||||
output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory"))
|
||||
fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness"
|
||||
with open(fitpath, "w") as fitfile:
|
||||
fitfile.write(str(ind.fitness))
|
||||
|
||||
# Log the fitness
|
||||
#logger.info("[%s] Fitness %s: %s" % (ind.environment_id, str(ind.fitness), str(ind)))
|
||||
|
||||
return ind.environment_id, ind.fitness
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines required global args for this plugin
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description='DNS plugin runner', allow_abbrev=False)
|
||||
parser.add_argument('--use-tcp', action='store_true', help='leverage TCP for this plugin')
|
||||
parser.add_argument('--environment-id', action='store', help="ID of the current environment")
|
||||
parser.add_argument('--output-directory', action='store', help="Where to output results")
|
||||
parser.add_argument('--port', action='store', type=int, default=53, help='port to use')
|
||||
args, _ = parser.parse_known_args(command)
|
||||
return vars(args)
|
|
@ -0,0 +1,463 @@
|
|||
"""
|
||||
Code influenced from:
|
||||
- https://github.com/emileaben/scapy-dns-ninja/blob/master/dns-ninja-server.py
|
||||
- https://thepacketgeek.com/scapy-p-09-scapy-and-dns/
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
# DNS Modules
|
||||
import dns.zone
|
||||
from dns.exception import DNSException
|
||||
|
||||
# Scapy modules
|
||||
from scapy.layers.dns import *
|
||||
from scapy.all import send
|
||||
|
||||
# Debugging
|
||||
from pprint import pformat
|
||||
|
||||
# TLDs
|
||||
from tld import get_fld
|
||||
|
||||
import inspect
|
||||
import random
|
||||
import os
|
||||
import sys
|
||||
|
||||
from plugins.plugin_server import ServerPlugin
|
||||
|
||||
# Listener - NetfilterQueue
|
||||
try:
|
||||
from netfilterqueue import NetfilterQueue
|
||||
except ImportError:
|
||||
print("ERROR: Failed to import netfilerqueue.")
|
||||
|
||||
# Listener - Socket
|
||||
import socket
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# Utils
|
||||
import datetime
|
||||
import actions.utils
|
||||
import logging
|
||||
|
||||
# Default values
|
||||
INTERFACE = "lo0"
|
||||
LISTENER = "socket_UDP"
|
||||
PORT = 53
|
||||
DNS_RESOLVER = "1.1.1.1"
|
||||
ZONES_DIR = "zones/"
|
||||
LOG_LEVEL = "info"
|
||||
|
||||
|
||||
class DNSServer(ServerPlugin):
|
||||
"""
|
||||
Purpose: Handle incoming DNS queries and respond with resource records defined in a zone configuration file (if
|
||||
exists for that domain) or respond with the answer given by a DNS resolver
|
||||
|
||||
Features:
|
||||
- Loads zone configuration files (--zones-dir)
|
||||
- Forwards DNS requests to a DNS resolver for domains that it does not know the answer to (--dns-resolver)
|
||||
- DNS forwarding can be disabled with (--no-forwarding)
|
||||
- Can act as the authority server for all DNS responses
|
||||
|
||||
Zones:
|
||||
- Support for A, MX, NS, TXT and CNAME
|
||||
- Other records may be automatically supported through the default action (no special case)
|
||||
- Only the first string per TXT record will be retrieved to avoid duplicated quotes
|
||||
|
||||
Logging:
|
||||
- Logs are created for each run and saved in the directory specified (--log-dir)
|
||||
- Logs can be disabled with (--no-log)
|
||||
|
||||
Python Test: tests/test_dns_server.py
|
||||
"""
|
||||
name = "dns"
|
||||
netfilter_queue = 'netfilterqueue'
|
||||
socket_UDP = 'socket_UDP'
|
||||
socket_TCP = 'socket_TCP'
|
||||
|
||||
def __init__(self, args, logger=None):
|
||||
"""
|
||||
Initializes the DNS Server.
|
||||
"""
|
||||
ServerPlugin.__init__(self)
|
||||
self.nfqueue = None
|
||||
self.nfqueue_num = None
|
||||
self.sock = None
|
||||
self.running = False
|
||||
self.zones = {}
|
||||
self.packet_counter = 0
|
||||
self.logger = logger
|
||||
|
||||
if not args:
|
||||
return
|
||||
|
||||
# Arguments
|
||||
self.interface = args["interface"]
|
||||
self.listener = args["listener"]
|
||||
self.port = args["port"]
|
||||
self.authority = args["authority"]
|
||||
self.resolver = args["dns_resolver"]
|
||||
self.zones_dir = args["zones_dir"]
|
||||
|
||||
def get_args(command):
|
||||
"""
|
||||
Sets up argparse and collects arguments.
|
||||
"""
|
||||
super_args = ServerPlugin.get_args(command)
|
||||
|
||||
parser = argparse.ArgumentParser(description='DNS Server')
|
||||
|
||||
# Network Configuration
|
||||
parser.add_argument('--interface', action='store', help='Interface to listen on', default=INTERFACE)
|
||||
parser.add_argument('--listener', action='store', choices=(DNSServer.socket_TCP, DNSServer.socket_UDP,
|
||||
DNSServer.netfilter_queue),
|
||||
help='Set the listener (Netfilterqueue is linux only)', default=DNSServer.socket_UDP)
|
||||
parser.add_argument('--port', type=int, action='store', help='DNS Server port to listen on', default=PORT)
|
||||
|
||||
# Zones
|
||||
parser.add_argument("--zones-dir", action='store', help="Zones directory", default=ZONES_DIR)
|
||||
|
||||
# Authority
|
||||
parser.add_argument('--authority', action='store_true', help='States that the DNS server is the authority server of'
|
||||
' all DNS responses')
|
||||
# DNS Resolver
|
||||
parser.add_argument('--dns-resolver', action='store', help="Specify a DNS resolver to forward DNS queries",
|
||||
default=DNS_RESOLVER)
|
||||
parser.add_argument('--no-forwarding', action='store_true', help='Disable forwarding DNS queries to a DNS resolver',
|
||||
default=False)
|
||||
|
||||
parser.add_argument('--log', action='store', choices=("debug", "info", "error"), help="Sets the log level",
|
||||
default=LOG_LEVEL)
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger):
|
||||
"""
|
||||
Starts the DNS Service
|
||||
"""
|
||||
self.running = True
|
||||
self.logger = logger
|
||||
# Setup the Listener
|
||||
if self.listener == DNSServer.netfilter_queue: # Netfilter Queue
|
||||
self.nfqueue_num = random.randint(11, 255)
|
||||
os.system(
|
||||
'iptables -t mangle -A PREROUTING -p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num)
|
||||
self.nfqueue = NetfilterQueue()
|
||||
self.nfqueue.bind(self.nfqueue_num, self.process_packet_netfilter)
|
||||
|
||||
elif self.listener == DNSServer.socket_UDP: # UDP Socket
|
||||
try:
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_UDP)
|
||||
self.sock.bind(('0.0.0.0', self.port))
|
||||
except socket.error as err:
|
||||
raise Exception("Error opening UDP socket")
|
||||
elif self.listener == DNSServer.socket_TCP: # TCP Socket
|
||||
try:
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.sock.bind(('0.0.0.0', self.port))
|
||||
except Exception as err:
|
||||
raise Exception("Error opening TCP socket")
|
||||
else: # None selected
|
||||
raise Exception("No listener has been selected")
|
||||
|
||||
# Load the DNS zones this server will support
|
||||
self.load_zones()
|
||||
self.logger.debug("Starting the DNS service")
|
||||
self.write_startup_file(args, logger)
|
||||
|
||||
# Netfilter
|
||||
if self.listener == DNSServer.netfilter_queue:
|
||||
try:
|
||||
self.nfqueue.run()
|
||||
except KeyboardInterrupt:
|
||||
os.system('iptables -t mangle -D PREROUTING '
|
||||
'-p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num)
|
||||
|
||||
# Socket UDP
|
||||
elif self.listener == DNSServer.socket_UDP:
|
||||
while True:
|
||||
try:
|
||||
data = self.sock.recv(1024)
|
||||
except socket.timeout:
|
||||
continue
|
||||
response_packet = self.build_response_packet(data)
|
||||
|
||||
if response_packet is not None:
|
||||
send(response_packet, verbose=0)#, iface=self.interface)
|
||||
|
||||
# Socket TCP
|
||||
elif self.listener == DNSServer.socket_TCP:
|
||||
self.sock.listen(10000)
|
||||
self.logger.debug("Socket is listening")
|
||||
|
||||
# Continuously accept new connections
|
||||
while True:
|
||||
try:
|
||||
connection, addr = self.sock.accept()
|
||||
|
||||
# Two byte length field
|
||||
message_length = connection.recv(2)
|
||||
if not message_length:
|
||||
continue
|
||||
|
||||
message_length = int(message_length[0]) * 256 + int(message_length[1])
|
||||
|
||||
# Receive the DNS contents
|
||||
dns_contents = connection.recv(message_length)
|
||||
if not dns_contents:
|
||||
continue
|
||||
|
||||
# Build response
|
||||
response_packet = self.build_response_packet(dns_contents, False)
|
||||
|
||||
if response_packet is not None:
|
||||
length = len(response_packet)
|
||||
connection.send(length.to_bytes(2, byteorder='big') + raw(response_packet))
|
||||
|
||||
connection.close()
|
||||
except KeyboardInterrupt:
|
||||
self.sock.close()
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def load_zones(self):
|
||||
"""
|
||||
Loads the DNS Zones in the zones directory specified (zones_dir)
|
||||
"""
|
||||
zones_dir = os.path.join(BASEPATH, self.zones_dir)
|
||||
self.logger.debug("Loading the DNS zones from %s", zones_dir)
|
||||
|
||||
# Each file in the zones directory is a domain
|
||||
for domain in os.listdir(zones_dir):
|
||||
try:
|
||||
self.zones[domain] = dns.zone.from_file(zones_dir + domain, domain, rdclass=1, relativize=False)
|
||||
self.logger.debug("Loaded zone: " + domain)
|
||||
except DNSException:
|
||||
self.logger.error("Error reading zone file:" + domain)
|
||||
|
||||
def forward_dns_query(self, packet: IP):
|
||||
"""
|
||||
Forwards the DNS query to a real DNS resolver and returns the DNS response
|
||||
"""
|
||||
|
||||
dns_response = sr1(
|
||||
IP(dst=self.resolver) /
|
||||
UDP(sport=5000, dport=53) /
|
||||
DNS(rd=1, id=packet[DNS].id, qd=packet[DNSQR]),
|
||||
verbose=0
|
||||
)
|
||||
|
||||
return dns_response[DNS]
|
||||
|
||||
def get_dns_query_info(self, packet: IP):
|
||||
"""
|
||||
Extract information from the DNS query
|
||||
"""
|
||||
question_name = packet[DNSQR].qname.decode("utf-8")
|
||||
question_type = dns.rdatatype.to_text(packet[DNSQR].qtype)
|
||||
|
||||
error = False
|
||||
|
||||
# Get the first level domain name (e.g. www.google.com -> google.com)
|
||||
domain_name = question_name[:-1] # Remove ending "."
|
||||
try:
|
||||
domain_name = get_fld(domain_name, fix_protocol=True)
|
||||
except Exception as e:
|
||||
self.logger.error("ERROR: Question Name: " + question_name + " - " + str(e))
|
||||
error = True
|
||||
|
||||
return question_name, domain_name, question_type, error
|
||||
|
||||
def get_resource_records(self, domain_name, question_name, question_type):
|
||||
"""
|
||||
Gets the appropriate resource record loaded earlier from the zone file
|
||||
"""
|
||||
resource_records = None
|
||||
|
||||
data = self.zones[domain_name].get_rdataset(question_name, question_type)
|
||||
|
||||
if data is None:
|
||||
# NXDOMAIN
|
||||
return resource_records, 0
|
||||
|
||||
# Build the resource records using scapy (DNSRR)
|
||||
for record in data:
|
||||
resource_record = DNSRR(rrname=question_name, type=question_type, ttl=data.ttl)
|
||||
resource_record_log = "Adding record: " + question_name + ' ' + str(data.ttl) + ' ' + question_type + ' '
|
||||
|
||||
if question_type == 'MX':
|
||||
resource_record_log += record.to_text()
|
||||
# DNS RDATA FORMAT: Preference (16 bit integer) + Exchange (DNS Name)
|
||||
resource_record[DNSRR].rdata = \
|
||||
struct.pack("!H", record.preference) + record.exchange.to_wire(None, None)
|
||||
elif question_type == 'TXT':
|
||||
# Retrieve only the first string in the TXT record to avoid duplicate quotes
|
||||
resource_record_log += dns.rdata._escapify(record.strings[0])
|
||||
resource_record[DNSRR].rdata = dns.rdata._escapify(record.strings[0])
|
||||
else:
|
||||
# Default: Records tested that work: A, NS, CNAME
|
||||
resource_record_log += record.to_text()
|
||||
resource_record[DNSRR].rdata = record.to_text()
|
||||
|
||||
#self.logger.debug(resource_record_log)
|
||||
|
||||
if resource_records is None:
|
||||
resource_records = resource_record
|
||||
else:
|
||||
resource_records = resource_records / resource_record
|
||||
|
||||
return resource_records, len(data)
|
||||
|
||||
def build_dns_response(self, packet):
|
||||
"""
|
||||
Build the DNS response packet using one of the following methods:
|
||||
1) Load the resource record(s) from a manually configured DNS zone file (if exists)
|
||||
OTHERWISE, if enabled:
|
||||
2) Send a DNS query to a DNS resolver and copy the DNS resource records
|
||||
"""
|
||||
|
||||
# Build response packet with empty DNS information and domain name error
|
||||
dns_response = DNS(id=packet[DNS].id, rcode=3, ra=1, qr=1, qdcount=1, ancount=0, qd=packet[DNS].qd)
|
||||
|
||||
# Extract information from the DNS query
|
||||
question_name, domain_name, question_type, dns_query_error = self.get_dns_query_info(packet)
|
||||
|
||||
info_log = "Query - Name: " + question_name + " | FLD: " + domain_name + \
|
||||
" | Record Type: " + question_type
|
||||
|
||||
if domain_name in self.zones and dns_query_error is False:
|
||||
# If we have a zone for this domain
|
||||
self.logger.debug("Found manually configured domain: " + domain_name)
|
||||
|
||||
# Get the resource records
|
||||
(resource_records, count) = self.get_resource_records(domain_name, question_name, question_type)
|
||||
|
||||
if count > 0:
|
||||
dns_response = DNS(id=packet[DNS].id, rcode=0, ra=1, qr=1, qdcount=1, ancount=count, qd=packet[DNS].qd,
|
||||
an=resource_records)
|
||||
|
||||
self.logger.debug(info_log + " | Action: Zone")
|
||||
|
||||
elif self.resolver is not None:
|
||||
# Forward the packet to a real DNS resolver
|
||||
self.logger.debug("No manually configured zone file for this domain; forwarding packet to " + self.resolver)
|
||||
dns_response = self.forward_dns_query(packet)
|
||||
self.logger.debug("Response from DNS resolver: " + pformat(dns_response))
|
||||
|
||||
self.logger.debug(info_log + " | Action: Forwarding")
|
||||
|
||||
if self.authority is True:
|
||||
dns_response[DNS].aa = 1
|
||||
|
||||
return dns_response
|
||||
|
||||
def process_packet_netfilter(self, listener_packet):
|
||||
"""
|
||||
Callback function for each packet received by netfilter
|
||||
"""
|
||||
if not self.running:
|
||||
return
|
||||
response_packet = self.build_response_packet(listener_packet)
|
||||
send(response_packet, verbose=0, iface=self.interface)
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stops this server.
|
||||
"""
|
||||
self.running = False
|
||||
|
||||
if self.listener == DNSServer.netfilter_queue:
|
||||
# Give the handlers two seconds to leave the callbacks before we forcibly unbind
|
||||
# the queues.
|
||||
time.sleep(2)
|
||||
self.nfqueue.unbind()
|
||||
os.system('iptables -t mangle -D PREROUTING '
|
||||
'-p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num)
|
||||
# Socket UDP
|
||||
elif self.listener == DNSServer.socket_UDP:
|
||||
if self.sock:
|
||||
self.sock.close()
|
||||
# Socket TCP
|
||||
elif self.listener == DNSServer.socket_TCP:
|
||||
if self.sock:
|
||||
self.sock.close()
|
||||
|
||||
ServerPlugin.stop(self)
|
||||
|
||||
def build_response_packet(self, listener_packet, raw_socket=True):
|
||||
"""
|
||||
Build the DNS response packet
|
||||
- If raw_socket is enabled include the Network and Transport Layer
|
||||
"""
|
||||
|
||||
packet = None
|
||||
|
||||
# Netfilter
|
||||
if self.listener == DNSServer.netfilter_queue:
|
||||
packet = IP(listener_packet.get_payload())
|
||||
listener_packet.drop()
|
||||
|
||||
# No transformations - UDP
|
||||
elif self.listener == DNSServer.socket_UDP:
|
||||
# Raw packet to scapy packet
|
||||
packet = IP(listener_packet)
|
||||
|
||||
# No transformations - TCP
|
||||
elif self.listener == DNSServer.socket_TCP:
|
||||
packet = DNS(listener_packet)
|
||||
|
||||
if packet is None or not packet.haslayer(DNS): # if this packet does not have DNS layer
|
||||
return None
|
||||
|
||||
#self.logger.debug("Received the following packet " + str(self.packet_counter + 1) + ": " + pformat(packet))
|
||||
|
||||
# Ignore DNS responses
|
||||
if packet[DNS].qr == 1:
|
||||
#self.logger.debug("Discarding DNS response packet\n")
|
||||
return None
|
||||
|
||||
self.packet_counter += 1
|
||||
|
||||
# Build DNS response packet
|
||||
dns_response = self.build_dns_response(packet)
|
||||
|
||||
#self.logger.debug(dns_response)
|
||||
|
||||
if raw_socket is True:
|
||||
response_packet = IP(dst=packet[IP].src, src=packet[IP].dst) / \
|
||||
UDP(dport=packet[UDP].sport, sport=packet[UDP].dport) / \
|
||||
dns_response
|
||||
|
||||
response_packet = IP(raw(response_packet))
|
||||
|
||||
else:
|
||||
response_packet = dns_response
|
||||
|
||||
#self.logger.debug("Response packet " + str(self.packet_counter) + ": " +
|
||||
# pformat(response_packet) + "\n")
|
||||
|
||||
return response_packet
|
||||
|
||||
|
||||
def main(args):
|
||||
"""
|
||||
Run the DNS server
|
||||
"""
|
||||
server = DNSServer(args)
|
||||
|
||||
if "dry_run" not in args:
|
||||
server.start()
|
||||
|
||||
return server
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(DNSServer.get_args(sys.argv[1:]))
|
|
@ -0,0 +1,26 @@
|
|||
$TTL 36000
|
||||
example.com. IN SOA ns1.example.com. hostmaster.example.com. (
|
||||
2005081201 ; serial
|
||||
28800 ; refresh (8 hours)
|
||||
1800 ; retry (30 mins)
|
||||
2592000 ; expire (30 days)
|
||||
86400 ) ; minimum (1 day)
|
||||
|
||||
example.com. 86400 NS ns1.example.com.
|
||||
example.com. 86400 NS ns2.example.com.
|
||||
example.com. 86400 MX 10 mail1.n2.example.com.
|
||||
example.com. 86400 MX 20 mail2.example.com.
|
||||
example.com. 86400 A 192.168.10.10
|
||||
example.com. 86400 A 192.168.10.11
|
||||
example.com. 86400 TXT "v=spf1 a:mail.example.com -all"
|
||||
|
||||
ns1.example.com. 86400 A 192.168.1.10
|
||||
ns1.example.com. 86400 A 192.168.1.11
|
||||
ns2.example.com. 86400 A 192.168.1.20
|
||||
mail.example.com. 86400 A 192.168.2.10
|
||||
mail2.example.com. 86400 A 192.168.2.20
|
||||
www2.example.com. 86400 A 192.168.10.20
|
||||
|
||||
www.example.com. 86400 CNAME example.com.
|
||||
ftp.example.com. 86400 CNAME example.com.
|
||||
webmail.example.com. 86400 CNAME example.com.
|
|
@ -0,0 +1,114 @@
|
|||
"""
|
||||
Client
|
||||
|
||||
Run by the evaluator, echo's data back and forth to the server
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.request
|
||||
|
||||
import requests
|
||||
|
||||
socket.setdefaulttimeout(1)
|
||||
|
||||
import actions.utils
|
||||
|
||||
from plugins.plugin_client import ClientPlugin
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class EchoClient(ClientPlugin):
|
||||
"""
|
||||
Defines the Echo client.
|
||||
"""
|
||||
name = "echo"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the echo client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines required args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='Echo Client')
|
||||
|
||||
parser.add_argument('--injection-censor', action='store_true', help="whether this censor injects content or sends RSTs to censor")
|
||||
parser.add_argument('--server', action='store', help="server to connect to")
|
||||
parser.add_argument('--http-request', action='store', help="send an HTTP get request with the given hostname to the echo server")
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to make a forbidden GET request to the server.
|
||||
"""
|
||||
fitness = 0
|
||||
port = int(args["port"])
|
||||
server = args["server"]
|
||||
bad_word = args["bad_word"]
|
||||
msg = bad_word
|
||||
if args.get("http_request"):
|
||||
msg = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % args.get("http_request")
|
||||
try:
|
||||
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
client.settimeout(10)
|
||||
client.connect((server, port))
|
||||
for idx in range(0, 5):
|
||||
if type(msg) == str:
|
||||
msg = msg.encode()
|
||||
client.sendall(msg)
|
||||
server_data = client.recv(1024)
|
||||
logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore'))
|
||||
if server_data == msg:
|
||||
fitness += 100
|
||||
elif server_data:
|
||||
fitness -= 90
|
||||
break
|
||||
|
||||
client.close()
|
||||
# If the fitness is 0, the strategy did something to corrupt/interfere with the socket
|
||||
# sending/receiving, usually by just artificially closing the connection. This behavior
|
||||
# should not be rewarded with a higher fitness
|
||||
if fitness == 0:
|
||||
fitness -= 100
|
||||
except socket.timeout:
|
||||
logger.debug("Client: Timeout")
|
||||
fitness -= 100
|
||||
except socket.error as exc:
|
||||
# If the censor we're running against tears down connects via RSTs, we can punish RSTs as
|
||||
# if the strategy did not harm the underlying connection. However, if the censor only injects
|
||||
# traffic, not resets, we should punish RSTs harshly, as the strategy likely caused it.
|
||||
|
||||
if exc.errno == 104:
|
||||
if args.get("injection_censor"):
|
||||
fitness -= 110
|
||||
else:
|
||||
fitness -= 90
|
||||
logger.debug("Client: Connection RST.")
|
||||
else:
|
||||
fitness -= 100
|
||||
logger.exception("Socket error caught in client echo test.")
|
||||
except Exception:
|
||||
logger.exception("Exception caught in client echo test.")
|
||||
fitness = -120
|
||||
finally:
|
||||
logger.debug("Client finished echo test.")
|
||||
return fitness * 4
|
|
@ -0,0 +1,95 @@
|
|||
import argparse
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
from plugins.plugin_server import ServerPlugin
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class EchoServer(ServerPlugin):
|
||||
"""
|
||||
Defines the Echo client.
|
||||
"""
|
||||
name = "echo"
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the Echo client.
|
||||
"""
|
||||
ServerPlugin.__init__(self)
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines arguments for this plugin
|
||||
"""
|
||||
super_args = ServerPlugin.get_args(command)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Echo Server')
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger):
|
||||
"""
|
||||
Initializes the Echo server.
|
||||
"""
|
||||
logger.debug("Echo server initializing")
|
||||
try:
|
||||
port = int(args["port"])
|
||||
control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
# Allow socket re-use
|
||||
control_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server_address = ('0.0.0.0', port)
|
||||
logger.debug("Binding to server address 0.0.0.0:%d" % port)
|
||||
control_socket.bind(server_address)
|
||||
control_socket.settimeout(5)
|
||||
control_socket.listen(1)
|
||||
except:
|
||||
logger.exception("Caught exception in echo run")
|
||||
return
|
||||
|
||||
try:
|
||||
connection, client_address = self.get_request(control_socket)
|
||||
if not connection:
|
||||
logger.error("Failed to get connection")
|
||||
return
|
||||
# Echo data back and forth
|
||||
for i in range(0, 5):
|
||||
data = connection.recv(1024)
|
||||
connection.sendall(data)
|
||||
|
||||
connection.close()
|
||||
except socket.error as e:
|
||||
if e.errno == 104:
|
||||
logger.debug("Server: Connection RST.")
|
||||
else:
|
||||
logger.debug("Server: Client quit.")
|
||||
except socket.ConnectionResetError:
|
||||
logger.debug("Server: Connection RST.")
|
||||
except Exception:
|
||||
logger.exception("Failed during server communication.")
|
||||
finally:
|
||||
logger.debug("Server exiting")
|
||||
|
||||
def get_request(self, control_socket):
|
||||
"""
|
||||
Get a request from the socket.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
sock, addr = control_socket.accept()
|
||||
sock.settimeout(5)
|
||||
return (sock, addr)
|
||||
except socket.timeout:
|
||||
pass
|
||||
return (None, None)
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stops this server.
|
||||
"""
|
||||
ServerPlugin.stop(self)
|
|
@ -0,0 +1,101 @@
|
|||
"""
|
||||
Run by the evaluator, tries to make a GET request to a given server
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.request
|
||||
|
||||
import requests
|
||||
|
||||
socket.setdefaulttimeout(1)
|
||||
|
||||
import external_sites
|
||||
import actions.utils
|
||||
|
||||
from plugins.plugin_client import ClientPlugin
|
||||
|
||||
BASEPATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class HTTPClient(ClientPlugin):
|
||||
"""
|
||||
Defines the HTTP client.
|
||||
"""
|
||||
name = "http"
|
||||
|
||||
def __init__(self, args):
|
||||
"""
|
||||
Initializes the HTTP client.
|
||||
"""
|
||||
ClientPlugin.__init__(self)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def get_args(command):
|
||||
"""
|
||||
Defines required args for this plugin
|
||||
"""
|
||||
super_args = ClientPlugin.get_args(command)
|
||||
parser = argparse.ArgumentParser(description='HTTP Client', prog="http/client.py")
|
||||
|
||||
parser.add_argument('--host-header', action='store', default="", help='specifies host header for HTTP request')
|
||||
parser.add_argument('--injected-http-contains', action='store', default="", help='checks if injected http response contains string')
|
||||
|
||||
args, _ = parser.parse_known_args(command)
|
||||
args = vars(args)
|
||||
|
||||
super_args.update(args)
|
||||
return super_args
|
||||
|
||||
def run(self, args, logger, engine=None):
|
||||
"""
|
||||
Try to make a forbidden GET request to the server.
|
||||
"""
|
||||
fitness = 0
|
||||
url = args.get("server", "")
|
||||
assert url, "Cannot launch HTTP test with no server"
|
||||
if not url.startswith("http://"):
|
||||
url = "http://" + url
|
||||
headers = {}
|
||||
if args.get('host_header'):
|
||||
headers["Host"] = args.get('host_header')
|
||||
|
||||
# If we've been given a non-standard port, append that to the URL
|
||||
port = args.get("port", 80)
|
||||
if port != 80:
|
||||
url += ":%s" % str(port)
|
||||
|
||||
url += "?q=%s" % args.get("bad_word")
|
||||
|
||||
injected_http = args.get("injected_http_contains")
|
||||
try:
|
||||
res = requests.get(url, allow_redirects=False, timeout=3, headers=headers)
|
||||
logger.debug(res.text)
|
||||
# If we need to monitor for an injected response, check that here
|
||||
if injected_http and injected_http in res.text:
|
||||
fitness -= 90
|
||||
else:
|
||||
fitness += 100
|
||||
except (requests.exceptions.ConnectionError, ConnectionResetError) as exc:
|
||||
logger.exception("Connection RST.")
|
||||
fitness -= 90
|
||||
except urllib.error.URLError as exc:
|
||||
logger.debug(exc)
|
||||
fitness += -101
|
||||
# Timeouts generally mean the strategy killed the TCP stream.
|
||||
# HTTPError usually mean the request was destroyed.
|
||||
# Punish this more harshly than getting caught by the censor.
|
||||
except (requests.exceptions.Timeout, requests.exceptions.HTTPError) as exc:
|
||||
logger.debug(exc)
|
||||
fitness += -120
|
||||
except Exception:
|
||||
logger.exception("Exception caught in HTTP test to site %s.", url)
|
||||
fitness += -100
|
||||
return fitness * 4
|
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue