From 5f13b926e32ee4375151e5adcfd6ba4f17bcb46c Mon Sep 17 00:00:00 2001 From: Kkevsterrr Date: Sun, 17 May 2020 10:15:16 -0400 Subject: [PATCH] Starting commit for documentation & genetic algorithm --- .coveragerc | 8 +- .travis.yml | 35 +- README.md | 130 ++- actions/action.py | 35 +- actions/drop.py | 11 + actions/duplicate.py | 10 + actions/fragment.py | 106 ++- actions/layer.py | 29 +- actions/packet.py | 5 +- actions/sleep.py | 15 +- actions/sniffer.py | 85 ++ actions/strategy.py | 141 +++ actions/tamper.py | 64 +- actions/trace.py | 13 +- actions/tree.py | 98 ++- actions/trigger.py | 52 +- actions/utils.py | 257 +++++- censors/censor.py | 163 ++++ censors/censor1.py | 86 ++ censors/censor10.py | 176 ++++ censors/censor11.py | 176 ++++ censors/censor2.py | 63 ++ censors/censor3.py | 98 +++ censors/censor4.py | 86 ++ censors/censor5.py | 87 ++ censors/censor6.py | 63 ++ censors/censor7.py | 74 ++ censors/censor8.py | 78 ++ censors/censor8b.py | 159 ++++ censors/censor9.py | 106 +++ censors/censor_driver.py | 94 ++ censors/censors.txt | 22 + censors/dummy.py | 26 + docker/Dockerfile | 17 + docker/README.md | 10 + docs/Makefile | 20 + docs/api/actions/action.rst | 7 + docs/api/actions/drop.rst | 7 + docs/api/actions/duplicate.rst | 7 + docs/api/actions/fragment.rst | 7 + docs/api/actions/layer.rst | 7 + docs/api/actions/packet.rst | 7 + docs/api/actions/sleep.rst | 7 + docs/api/actions/strategy.rst | 7 + docs/api/actions/tamper.rst | 7 + docs/api/actions/trace.rst | 7 + docs/api/actions/tree.rst | 7 + docs/api/actions/trigger.rst | 7 + docs/api/actions/utils.rst | 7 + docs/api/engine.rst | 7 + docs/api/evaluator.rst | 7 + docs/api/evolve.rst | 7 + docs/api/plugins/dns.rst | 17 + docs/api/plugins/echo.rst | 12 + docs/api/plugins/http.rst | 17 + docs/api/plugins/plugin.rst | 7 + docs/api/plugins/plugin_client.rst | 7 + docs/api/plugins/plugin_server.rst | 7 + docs/api/plugins/sni.rst | 7 + docs/conf.py | 78 ++ docs/extending/actions.rst | 127 +++ docs/extending/contributing.rst | 6 + docs/extending/layers.rst | 2 + docs/extending/plugins.rst | 276 ++++++ docs/howitworks/addingaworker.rst | 42 + docs/howitworks/engine.rst | 37 + docs/howitworks/evaluation.rst | 70 ++ docs/howitworks/evaluator.rst | 268 ++++++ docs/howitworks/evolution.rst | 42 + docs/howitworks/howitworks.rst | 86 ++ docs/howitworks/logging.rst | 28 + docs/howitworks/puttingittogether.rst | 21 + docs/howitworks/testing.rst | 13 + docs/index.rst | 45 + docs/intro/gettingstarted.rst | 93 ++ docs/intro/introduction.rst | 28 + docs/intro/setup.rst | 49 ++ docs/make.bat | 35 + engine.py | 196 ++++- evaluator.py | 1156 +++++++++++++++++++++++++ evolve.py | 834 ++++++++++++++++++ library.py | 480 ++++++++++ plugins/discard/client.py | 171 ++++ plugins/discard/server.py | 96 ++ plugins/dns/client.py | 122 +++ plugins/dns/plugin.py | 143 +++ plugins/dns/server.py | 463 ++++++++++ plugins/dns/zones/example.com | 26 + plugins/echo/client.py | 114 +++ plugins/echo/server.py | 95 ++ plugins/http/client.py | 101 +++ plugins/http/external_sites.py | 3 + plugins/http/plugin.py | 263 ++++++ plugins/http/server.py | 69 ++ plugins/plugin.py | 22 + plugins/plugin_client.py | 130 +++ plugins/plugin_server.py | 207 +++++ plugins/sni/client.py | 89 ++ requirements.txt | 17 +- tests/DNS/zones/example.com | 26 + tests/DNS/zones/example2.com | 26 + tests/common.py | 84 ++ tests/conftest.py | 98 +++ tests/pytest.ini | 2 + tests/test_censors.py | 79 ++ tests/test_compress.py | 26 + tests/test_dns_server.py | 443 ++++++++++ tests/test_duplicate.py | 24 + tests/test_engine.py | 51 +- tests/test_evaluator.py | 844 ++++++++++++++++++ tests/test_evolve.py | 484 +++++++++++ tests/test_fragment.py | 183 +++- tests/test_library.py | 37 + tests/test_options.py | 138 +++ tests/test_packet.py | 36 +- tests/test_parse.py | 77 ++ tests/test_plugins.py | 52 ++ tests/test_population_files.py | 166 ++++ tests/test_sleep.py | 11 +- tests/test_strategy.py | 147 +++- tests/test_tamper.py | 120 ++- tests/test_trace.py | 49 +- tests/test_tree.py | 147 +++- tests/test_trigger.py | 48 +- tests/test_utils.py | 44 +- workers/example/example.pem | 27 + workers/example/example.pub | 1 + workers/example/worker.json | 16 + 128 files changed, 12249 insertions(+), 264 deletions(-) create mode 100644 actions/sniffer.py create mode 100644 censors/censor.py create mode 100644 censors/censor1.py create mode 100644 censors/censor10.py create mode 100644 censors/censor11.py create mode 100644 censors/censor2.py create mode 100644 censors/censor3.py create mode 100644 censors/censor4.py create mode 100644 censors/censor5.py create mode 100644 censors/censor6.py create mode 100644 censors/censor7.py create mode 100644 censors/censor8.py create mode 100644 censors/censor8b.py create mode 100644 censors/censor9.py create mode 100644 censors/censor_driver.py create mode 100644 censors/censors.txt create mode 100644 censors/dummy.py create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docs/Makefile create mode 100644 docs/api/actions/action.rst create mode 100644 docs/api/actions/drop.rst create mode 100644 docs/api/actions/duplicate.rst create mode 100644 docs/api/actions/fragment.rst create mode 100644 docs/api/actions/layer.rst create mode 100644 docs/api/actions/packet.rst create mode 100644 docs/api/actions/sleep.rst create mode 100644 docs/api/actions/strategy.rst create mode 100644 docs/api/actions/tamper.rst create mode 100644 docs/api/actions/trace.rst create mode 100644 docs/api/actions/tree.rst create mode 100644 docs/api/actions/trigger.rst create mode 100644 docs/api/actions/utils.rst create mode 100644 docs/api/engine.rst create mode 100644 docs/api/evaluator.rst create mode 100644 docs/api/evolve.rst create mode 100644 docs/api/plugins/dns.rst create mode 100644 docs/api/plugins/echo.rst create mode 100644 docs/api/plugins/http.rst create mode 100644 docs/api/plugins/plugin.rst create mode 100644 docs/api/plugins/plugin_client.rst create mode 100644 docs/api/plugins/plugin_server.rst create mode 100644 docs/api/plugins/sni.rst create mode 100644 docs/conf.py create mode 100644 docs/extending/actions.rst create mode 100644 docs/extending/contributing.rst create mode 100644 docs/extending/layers.rst create mode 100644 docs/extending/plugins.rst create mode 100644 docs/howitworks/addingaworker.rst create mode 100644 docs/howitworks/engine.rst create mode 100644 docs/howitworks/evaluation.rst create mode 100644 docs/howitworks/evaluator.rst create mode 100644 docs/howitworks/evolution.rst create mode 100644 docs/howitworks/howitworks.rst create mode 100644 docs/howitworks/logging.rst create mode 100644 docs/howitworks/puttingittogether.rst create mode 100644 docs/howitworks/testing.rst create mode 100644 docs/index.rst create mode 100644 docs/intro/gettingstarted.rst create mode 100644 docs/intro/introduction.rst create mode 100644 docs/intro/setup.rst create mode 100644 docs/make.bat create mode 100644 evaluator.py create mode 100644 evolve.py create mode 100644 library.py create mode 100644 plugins/discard/client.py create mode 100644 plugins/discard/server.py create mode 100644 plugins/dns/client.py create mode 100644 plugins/dns/plugin.py create mode 100644 plugins/dns/server.py create mode 100644 plugins/dns/zones/example.com create mode 100644 plugins/echo/client.py create mode 100644 plugins/echo/server.py create mode 100644 plugins/http/client.py create mode 100755 plugins/http/external_sites.py create mode 100644 plugins/http/plugin.py create mode 100644 plugins/http/server.py create mode 100644 plugins/plugin.py create mode 100644 plugins/plugin_client.py create mode 100644 plugins/plugin_server.py create mode 100644 plugins/sni/client.py create mode 100644 tests/DNS/zones/example.com create mode 100644 tests/DNS/zones/example2.com create mode 100644 tests/common.py create mode 100644 tests/conftest.py create mode 100644 tests/pytest.ini create mode 100644 tests/test_censors.py create mode 100644 tests/test_compress.py create mode 100644 tests/test_dns_server.py create mode 100644 tests/test_duplicate.py create mode 100644 tests/test_evaluator.py create mode 100644 tests/test_evolve.py create mode 100644 tests/test_library.py create mode 100644 tests/test_options.py create mode 100644 tests/test_parse.py create mode 100644 tests/test_plugins.py create mode 100644 tests/test_population_files.py create mode 100644 workers/example/example.pem create mode 100644 workers/example/example.pub create mode 100644 workers/example/worker.json diff --git a/.coveragerc b/.coveragerc index 4370e6b..a8029dc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -9,8 +9,12 @@ exclude_lines = raise NotImplementedError if __name__ == .__main__.: def get_args - def main + except KeyboardInterrupt: ignore_errors = True omit = + experimental/* + experimental/smtp/* tests/* - examples/* + censors/* + censor_driver.py + graph.py diff --git a/.travis.yml b/.travis.yml index 9e296ac..2d2cea2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,39 +1,32 @@ sudo: required dist: "bionic" +services: + - docker + language: python python: - "3.6" install: - # Travis recently added systemd-resolvd to their VMs. Since full Geneva often runs its own DNS - # server to test DNS strategies, we need to disable system-resolvd. - # First disable the service - - sudo systemctl disable systemd-resolved.service - # Stop the service - - sudo systemctl stop systemd-resolved - # With systemd not running, our own hostname won't resolve - this causes issues with sudo. - # Add back our hostname to /etc/hosts/ so sudo does not complain - - echo $(hostname -I | cut -d\ -f1) $(hostname) | sudo tee -a /etc/hosts - # Replace the 127.0.0.53 nameserver with Google's - - sudo sed 's/nameserver.*/nameserver 8.8.8.8/' /etc/resolv.conf > /tmp/resolv.conf.new + # Travis recently added systemd-resolvd to their VMs. We must disable this to test our own DNS server + - sudo systemctl disable systemd-resolved.service # First disable the service + - sudo systemctl stop systemd-resolved # Stop the service + - echo $(hostname -I | cut -d\ -f1) $(hostname) | sudo tee -a /etc/hosts # add back our hostname to /etc/hosts so sudo does not complain + - sudo sed 's/nameserver.*/nameserver 8.8.8.8/' /etc/resolv.conf > /tmp/resolv.conf.new # replace the 127.0.0.53 nameserver with Google's - sudo mv /tmp/resolv.conf.new /etc/resolv.conf - # Now that systemd-resolv.conf is safely disabled, we can now setup for Geneva + - sudo systemctl start docker # must now restart docker so the resolv.conf changes propagate to its containers - sudo apt-get clean # travis having mirror sync issues - # Install dependencies - sudo apt-get update - sudo apt-get -y install libnetfilter-queue-dev python3 python3-pip python3-setuptools graphviz - # Since sudo is required but travis does not set up the root environment, we must override the - # secure_path in sudoers in order for travis's setup to take effect for sudo commands - - printf "Defaults\tenv_reset\nDefaults\tmail_badpass\nDefaults\tsecure_path="/home/travis/virtualenv/python3.6.7/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"\nroot\tALL=(ALL:ALL) ALL\n#includedir /etc/sudoers.d\n" > /tmp/sudoers.tmp - # Verify the sudoers file - - sudo visudo -c -f /tmp/sudoers.tmp - # Copy in the sudoers file - - sudo cp /tmp/sudoers.tmp /etc/sudoers - # Now that sudo is good to go, finish installing dependencies + - printf "Defaults\tenv_reset\nDefaults\tmail_badpass\nDefaults\tsecure_path="/home/travis/virtualenv/python3.6.7/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"\nroot\tALL=(ALL:ALL) ALL\n#includedir /etc/sudoers.d\n" > /tmp/sudoers.tmp # Since sudo is required but travis does not set up the root environment, we must override the secure_path in sudoers in order for travis's setup to take effect for sudo commands + - sudo visudo -c -f /tmp/sudoers.tmp # Verify the sudoers file + - sudo cp /tmp/sudoers.tmp /etc/sudoers # Copy in the sudoers file + - sudo echo $PATH # Confirm the PATH changes took effect - sudo python3 -m pip install -r requirements.txt - sudo python3 -m pip install slackclient pytest-cov + - docker build -t base:latest -f docker/Dockerfile . script: - sudo python3 -m pytest --cov=./ -sv tests/ --tb=short diff --git a/README.md b/README.md index 14075b2..70dbae2 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,17 @@ # Geneva [![Build Status](https://travis-ci.com/Kkevsterrr/geneva.svg?branch=master)](https://travis-ci.com/Kkevsterrr/geneva) [![codecov](https://codecov.io/gh/Kkevsterrr/geneva/branch/master/graph/badge.svg)](https://codecov.io/gh/Kkevsterrr/geneva) -Geneva is an artificial intelligence tool that defeats censorship by exploiting bugs in censors, such as those in China, India, and Kazakhstan. Unlike many other anti-censorship solutions which require assistance from outside the censoring regime (Tor, VPNs, etc.), Geneva runs strictly on the client. +Geneva is an artificial intelligence tool that defeats censorship by exploiting bugs in censors, such as those in China, India, and Kazakhstan. Unlike many other anti-censorship solutions which require assistance from outside the censoring regime (Tor, VPNs, etc.), Geneva runs strictly on one side of the connection (either the client or server side). -Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion strategies and has found several previously unknown bugs in censors. Geneva's strategies manipulate the client's packets to confuse the censor without impacting the client/server communication. This makes Geneva effective against many types of in-network censorship (though it cannot be used against IP-blocking censorship). +Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion strategies and has found several previously unknown bugs in censors. Geneva's strategies manipulate the network stream to confuse the censor without impacting the client/server communication. This makes Geneva effective against many types of in-network censorship (though it cannot be used against IP-blocking censorship). -This code release specifically contains the strategy engine used by Geneva, its Python API, and a subset of published strategies, so users and researchers can test and deploy Geneva's strategies. To learn more about how Geneva works, visit [How it Works](#How-it-Works). We will be releasing the genetic algorithm at a later date. +Geneva is composed of two high level components: its genetic algorithm (which it uses to evolve new censorship evasion strategies) and its strategy engine (which is uses to run an individual censorship evasion strategy over a network connection). + +This codebase contains the Geneva's full implementation: its genetic algorithm, strategy engine, Python API, and a subset of published strategies. With these tools, users and researchers alike can evolve new strategies or leverage existing strategies to evade censorship. To learn more about how Geneva works, see [How it Works](#How-it-Works). ## Setup Geneva has been developed and tested for Centos or Debian-based systems. Due to limitations of -netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6* (with more versions coming soon). +netfilter and raw sockets, Geneva does not work on OS X or Windows at this time and requires *python3.6*. Install netfilterqueue dependencies: ``` @@ -21,7 +23,10 @@ Install Python dependencies: # python3 -m pip install -r requirements.txt ``` -## Running it +## Running a Strategy + +A censorship evasion strategy is simply a _description of how network traffic should be modified_. A strategy is not +code, it is a description that tells the engine how it should operate over traffic. For a fuller description of the DNA syntax, see [Censorship Evasion Strategies](#Censorship-Evasion-Strategies). ``` # python3 engine.py --server-port 80 --strategy "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:corrupt},),)-|" --log debug @@ -38,27 +43,67 @@ this will fail. To fix this, remove those rules. ## Strategy Library -Geneva has found dozens of strategies that work against censors in China, Kazakhstan, and India. We include several of these strategies in [strategies.md](strategies.md). Note that this file contains success rates for each individual country; a strategy that works in one country may not work as well as other countries. +Geneva has found dozens of strategies that work against censors in China, Kazakhstan, India, and Iran. We include several of these strategies in [strategies.md](strategies.md). Note that this file contains success rates for each individual country; a strategy that works in one country may not work as well as other countries. Researchers have observed that strategies may have differing success rates based on your exact location. Although we have not observed this from our vantage points, you may find that some strategies may work differently in a country we have tested. If this is the case, don't be alarmed. However, please feel free to reach out to a member of the team directly or open an issue on this page so we can track how the strategies work from other geographic locations. ## Disclaimer -Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it. +Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it. ------- -## How it Works +# How it Works -See our paper for an in-depth read on how Geneva works. Below is a rundown of the format of Geneva's strategy DNA. +See our [paper](#Paper) for an in-depth read on how Geneva works. Below is a walkthrough of the main concepts behind Geneva, the major components of the codebase, and how they can be used. + +## Censorship Evasion Strategies + +A censorship evasion strategy is simply a _description of how network traffic should be modified_. A strategy is _not +code_, it is a description that tells Geneva's stratgy engine how it should manipulate network traffic. The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted. + +A censorship evasion strategy composed of one or more packet-level building blocks. Geneva's core building blocks are: +1. `duplicate`: takes one packet and returns two copies of the packet +2. `drop`: takes one packet and returns no packets (drops the packet) +3. `tamper`: takes one packet and returns the modified packet +4. `fragment`: takes one packet and returns two fragments or two segments + +Since `duplicate` and `fragment` introduce _branching_, these actions are composed into a binary-tree structure called an _action tree_. Each tree also has a _trigger_. The trigger describes which packets the tree should run on, and the tree describes what should happen to each of those packets when the trigger fires. Once a trigger fires on a packet, it pulls the packet into the tree for modifications, and the packets that emerge from the tree are sent on the wire. Recall that Geneva operates at the packet level, therefore all triggers are packet-level triggers. + +Multiple action trees together form a _forest_. Geneva handles outbound and inbound packets differently, so strategies are composed of two forests: an outbound forest and an inbound forest. + +Consider the following example of a simple Geneva strategy. +``` + +---------------+ + | TCP:flags:A | <-- triggers on TCP packets with the flags field set to 'ACK' + +-------+-------+ matching packets are captured and pulled into the tree + | + +---------v---------+ + duplicate <-- makes two copies of the given packet. the tree is processed + +---------+---------+ with an inorder traversal, so the left side is run first + | + +-------------+------------+ + | | ++------------v----------+ v <-- dupilcate has no right child, so this packet will be sent on the wire unimpacted + tamper + {TCP:flags:replace:R} <-- parameters to this action describe how the packet should be tampered ++------------+----------+ + | ++------------v----------+ + tamper + {TCP:chksum:corrupt} ++------------+----------+ + | + v <-- packets that emerge from an in-order traversal of the leaves are sent on the wire + +``` + +This strategy triggers on `TCP` packets with the `flags` field set to `ACK`. It makes a duplicate of the `ACK` packet; the first duplicate has its flags field changed to `RST` and its checksum (`chksum`) field corrupted; the second duplicate is unchaged. Both packets are then sent on the network. ### Strategy DNA -Geneva's strategies can be arbitrarily complicated, and it defines a well-formatted syntax for -expressing strategies to the engine. - -A strategy is simply a _description of how network traffic should be modified_. A strategy is not -code, it is a description that tells the engine how it should operate over traffic. +These strategies can be arbitrarily complicated, and Geneva defines a well-formatted string syntax for +unambiguously expressing strategies. A strategy divides how it handles outbound and inbound packets: these are separated in the DNA by a "\\/". Specifically, the strategy format is ` \/ `. If `\/` is not @@ -66,35 +111,50 @@ present in a strategy, all of the action trees are in the outbound forest. Both forests are composed of action trees, and each forest is allowed an arbitrarily many trees. -An action tree is comprised of a _trigger_ and a _tree_. The trigger describes _when_ the strategy -should run, and the tree describes what should happen when the trigger fires. Recall that Geneva -operates at the packet level, therefore all triggers are packet-level triggers. Action trees start -with a trigger, and always end with a `-|`. +Action trees always start with a trigger, which is formatted as: `[::]`. For example, the trigger: `[TCP:flags:S]` will run its corresponding tree whenever it sees a `TCP` packet with the `flags` field set to `SYN`. If the corresponding action tree is `[TCP:flags:S]-drop-|`, this action tree will cause the engine to drop any `SYN` packets. `[TCP:flags:S]-duplicate-|` will cause the engine to duplicate any SYN packets. Triggers also can contain an optional 4th parameter for _gas_, which describes the number of times a trigger can fire. The triger `[IP:version:4:4]` will run only on the first 4 IPv4 packets it sees. If the gas is negative, the trigger acts as a _bomb_ trigger, which means the trigger will not fire until a certain number of applicable packets have been seen. For example, the trigger `[IP:version:4:-2]` will trigger only after it has seen two matching packets (and it will not trigger on those first packets). -Triggers operate as exact-matches, are formatted as follows: `[::]`. For -example, the trigger: `[TCP:flags:S]` will run its corresponding tree whenever it sees a `SYN` -TCP packet. If the corresponding action tree is `[TCP:flags:S]-drop-|`, this action tree will cause -the engine to drop any `SYN` packets. `[TCP:flags:S]-duplicate-|` will cause the engine to -duplicate the SYN packet. +Syntactically, action trees end with `-|`. -Depending on the type of action, some actions can have up to two children. These are represented +Depending on the type of action, some actions can have up to two children (such as `duplicate`). These are represented with the following syntax: `[TCP:flags:S]-duplicate(,)-|`, where `` and `` themselves are trees. If `(,)` is not specified, any packets -that emerge from the action will be sent on the wire. +that emerge from the action will be sent on the wire. If an action only has one child (such as `tamper`), it is always the left child. `[TCP:flags:S]-tamper{}(,)-|` -Any action that has parameters associated with it contain those parameters in `{}`. Consider the -following strategy with `tamper`. -``` -[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R},)-| \/ -``` -This strategy takes outbound `ACK` packets and duplicates them. To the first duplicate, it tampers -the packet by replacing the `TCP` `flags` field with `RST`, and does nothing to the second -duplicate. +Actions that have parameters specify those parameters within `{}`. For example, giving parameters to the `tamper` action could look like: `[TCP:flags:S]-tamper{TCP:flags:replace:A}-|`. This strategy would trigger on TCP `SYN` packets and replace the TCP `flags` field to `ACK`. -Note that due to NFQueue limitations, actions that introduce branching (fragment, duplicate) are +Putting this all together, below is the strategy DNA representation of the above diagram: +``` +[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/ +``` + +Geneva has code to parse this strategy DNA into strategies that can be applied to network traffic using the engine. + +Note that due to limitations of Scapy and NFQueue, actions that introduce branching (`fragment`, `duplicate`) are disabled for incoming action forests. -------- +## Engine + +The strategy engine (`engine.py`) applies a strategy to a network connection. The engine works by capturing all traffic to/from a specified port. Packets that match an active trigger are run through the associated action-tree, and packets that emerge from the tree are sent on the wire. + +The engine also has a Python API for using it in your application. It can be used as a context manager or invoked in the background as a thread. +For example, consider the following simple application. + +```python +import os +import engine + +# Port to run the engine on +port = 80 +# Strategy to use +strategy = "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/" + +# Create the engine in debug mode +with engine.Engine(port, strategy, log_level="debug") as eng: + os.system("curl http://example.com?q=ultrasurf") +``` +This script creates an instance of the engine with a specified strategy, and that strategy will be running for everything within the context manager. When the context manager exits, the engine will clean itself up. See the `examples/` folder for more use cases of the engine. + +Due to limitations of scapy and NFQueue, the engine cannot be used to communicate with localhost. ## Citation diff --git a/actions/action.py b/actions/action.py index 03e6d4f..69d1e97 100644 --- a/actions/action.py +++ b/actions/action.py @@ -1,7 +1,5 @@ """ -Action - -Geneva object for defining a packet-level action. +Geneva superclass object for defining a packet-level action. """ import inspect @@ -17,16 +15,24 @@ ACTION_CACHE["in"] = {} ACTION_CACHE["out"] = {} BASEPATH = os.path.sep.join(os.path.dirname(os.path.abspath(__file__)).split(os.path.sep)[:-1]) + class Action(): """ Defines the superclass for a Geneva Action. """ # Give each Action a unique ID - this is needed for graphing/visualization ident = 0 + # Each Action has a 'frequency' field - this defines how likely it is to be chosen + # when a new action is chosen + frequency = 0 def __init__(self, action_name, direction): """ Initializes this action object. + + Args: + action_name (str): Name of this action ("duplicate") + direction (str): Direction of this action ("out", "both", "in") """ self.enabled = True self.action_name = action_name @@ -45,6 +51,12 @@ class Action(): """ Returns whether this action applies to the given direction, as branching actions are not supported on inbound trees. + + Args: + direction (str): Direction to check if this action applies ("out", "in", "both") + + Returns: + bool: whether or not this action can be used to a given direction """ if direction == self.direction or self.direction == "both": return True @@ -67,6 +79,14 @@ class Action(): Dynamically imports all of the Action classes in this directory. Will only return terminal actions if terminal is set to True. + + Args: + direction (str): Limit imported actions to just those that can run to this direction ("out", "in", "both") + disabled (list, optional): list of actions that are disabled + allow_terminal (bool): whether or not terminal actions ("drop") should be imported + + Returns: + dict: Dictionary of imported actions """ if disabled is None: disabled = [] @@ -84,7 +104,6 @@ class Action(): else: return ACTION_CACHE[direction][terminal] - collected_actions = [] # Get the base path for the project relative to this file path = os.path.join(BASEPATH, "actions") @@ -117,6 +136,14 @@ class Action(): def parse_action(str_action, direction, logger): """ Parses a string action into the action object. + + Args: + str_action (str): String representation of an action to parse + direction (str): Limit actions searched through to just those that can run to this direction ("out", "in", "both") + logger (:obj:`logging.Logger`): a logger to log with + + Returns: + :obj:`action.Action`: A parsed action object """ # Collect all viable actions that can run for each respective direction outs = Action.get_actions("out") diff --git a/actions/drop.py b/actions/drop.py index a17436b..293a2b7 100644 --- a/actions/drop.py +++ b/actions/drop.py @@ -1,7 +1,18 @@ from actions.action import Action class DropAction(Action): + """ + Geneva action to drop the given packet. + """ + frequency = 1 + def __init__(self, environment_id=None): + """ + Initializes this drop action. + + Args: + environment_id (str, optional): Environment ID of the strategy we are a part of + """ Action.__init__(self, "drop", "both") self.terminal = True self.branching = False diff --git a/actions/duplicate.py b/actions/duplicate.py index 17036a6..0434ce5 100644 --- a/actions/duplicate.py +++ b/actions/duplicate.py @@ -2,6 +2,10 @@ from actions.action import Action class DuplicateAction(Action): + """ + Defines the DuplicateAction - returns two copies of the given packet. + """ + frequency = 3 def __init__(self, environment_id=None): Action.__init__(self, "duplicate", "out") self.branching = True @@ -13,3 +17,9 @@ class DuplicateAction(Action): """ logger.debug(" - Duplicating given packet %s" % str(packet)) return packet, packet.copy() + + def mutate(self, environment_id=None): + """ + Swaps its left and right child + """ + self.left, self.right = self.right, self.left diff --git a/actions/fragment.py b/actions/fragment.py index d7f80c3..9bd9b28 100644 --- a/actions/fragment.py +++ b/actions/fragment.py @@ -5,19 +5,32 @@ import actions.packet from scapy.all import IP, TCP, fragment +MAX_UINT = 4294967295 + + class FragmentAction(Action): - def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True): - ''' - correct_order specifies if the fragmented packets should come in the correct order - fragsize specifies how - ''' + """ + Defines the FragmentAction for Geneva - fragments or segments the given packet. + """ + frequency = 2 + def __init__(self, environment_id=None, correct_order=None, fragsize=-1, segment=True, overlap=0): + """ + Initializes a fragment action object. + + Args: + environment_id (str, optional): Environment ID of the strategy this object is a part of + correct_order (bool, optional): Whether or not the fragments/segments should be returned in the correct order + fragsize (int, optional): The index this packet should be cut. Defaults to -1, which cuts it in half. + segment (bool, optional): Whether we should perform fragmentation or segmentation + overlap (int, optional): How many bytes the fragments/segments should overlap + """ Action.__init__(self, "fragment", "out") self.enabled = True self.branching = True self.terminal = False self.fragsize = fragsize self.segment = segment - + self.overlap = overlap if correct_order == None: self.correct_order = self.get_rand_order() else: @@ -87,6 +100,9 @@ class FragmentAction(Action): Segments a packet into two, given the size of the first packet (0:fragsize) Always returns two packets, since fragment is a branching action, so if we are unable to segment, it will duplicate the packet. + + If overlap is specified, it will select n bytes from the second packet + and append them to the first, and increment the sequence number accordingly """ if not packet.haslayer("TCP") or not hasattr(packet["TCP"], "load") or not packet["TCP"].load: return packet, packet.copy() # duplicate if no TCP or no payload to segment @@ -101,7 +117,11 @@ class FragmentAction(Action): fragsize = int(len(payload)/2) # Craft new packets - pkt1 = IP(packet["IP"])/payload[:fragsize] + + # Make sure we don't go out of bounds by choosing the min + overlap_bytes = min(len(payload[fragsize:]), self.overlap) + # Attach these bytes to the first packet + pkt1 = IP(packet["IP"])/payload[:fragsize + overlap_bytes] pkt2 = IP(packet["IP"])/payload[fragsize:] # We cannot rely on scapy's native parsing here - if a previous action has changed the @@ -116,7 +136,11 @@ class FragmentAction(Action): packet2 = actions.packet.Packet(pkt2) # Reset packet2's SYN number - packet2["TCP"].seq += fragsize + if packet2["TCP"].seq + fragsize > MAX_UINT: + # Wrap sequence numbers around if greater than MAX_UINT + packet2["TCP"].seq = packet2["TCP"].seq + fragsize - MAX_UINT - 1 + else: + packet2["TCP"].seq += fragsize del packet1["IP"].chksum del packet2["IP"].chksum @@ -147,10 +171,14 @@ class FragmentAction(Action): Returns a string representation with the fragsize """ s = Action.__str__(self) - if self.segment: - s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + "}" + if not self.overlap: + ending = "}" else: - s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + "}" + ending = ":" + str(self.overlap) + "}" + if self.segment: + s += "{" + "tcp" + ":" + str(self.fragsize) + ":" + str(self.correct_order) + ending + else: + s += "{" + "ip" + ":"+ str(self.fragsize) + ":" + str(self.correct_order) + ending return s def parse(self, string, logger): @@ -169,22 +197,36 @@ class FragmentAction(Action): num_parameters = string.count(":") # If num_parameters is greater than 2, it's not a valid fragment action - if num_parameters != 2: - msg = "Cannot parse fragment action %s" % string - logger.error(msg) - raise Exception(msg) - else: + if num_parameters == 2: params = string.split(":") seg, fragsize, correct_order = params + overlap = 0 if "tcp" in seg: self.segment = True else: self.segment = False + elif num_parameters == 3: + params = string.split(":") + seg, fragsize, correct_order, overlap = params + if overlap.endswith("}"): + overlap = overlap[:-1] # Chop off trailing } + if "tcp" in seg: + self.segment = True + else: + self.segment = False + + else: + msg = "Cannot parse fragment action %s" % string + logger.error(msg) + raise Exception(msg) + try: # Try to convert to int self.fragsize = int(fragsize) - except ValueError: + self.overlap = int(overlap) + except ValueError as e: + print(e) msg = "Cannot parse fragment action %s" % string logger.error(msg) raise Exception(msg) @@ -196,3 +238,33 @@ class FragmentAction(Action): self.correct_order = False return True + + def mutate(self, environment_id=None): + """ + Mutates the fragment action - it either chooses a new segment offset, + switches the packet order, and/or changes whether it segments or fragments. + """ + self.correct_order = self.get_rand_order() + self.segment = random.choice([True, True, True, False]) + if self.segment: + if random.random() < 0.5: + self.fragsize = int(random.uniform(1, 60)) + else: + self.fragsize = -1 + else: + if random.random() < 0.2: + self.fragsize = int(random.uniform(1, 50)) + else: + self.fragsize = -1 + + if random.random() < .5: + # Somewhat aggressively overlap + if random.random() < .5: + if self.fragsize == -1: + self.overlap = 5 + else: + self.overlap = int(self.fragsize/2) + else: + self.overlap = int(random.uniform(1, 50)) + + return self diff --git a/actions/layer.py b/actions/layer.py index 1f16991..2fd3075 100644 --- a/actions/layer.py +++ b/actions/layer.py @@ -1,4 +1,5 @@ import binascii +import copy import random import string import os @@ -6,7 +7,6 @@ import urllib.parse from scapy.all import IP, RandIP, UDP, DNS, DNSQR, Raw, TCP, fuzz - class Layer(): """ Base class defining a Geneva packet layer. @@ -180,8 +180,7 @@ class Layer(): value = urllib.parse.unquote(value) value = value.encode('utf-8') - # Add support for injecting arbitrary protocol payloads if requested - dns_payload = b"\x009ib\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x08examples\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00\x01\x00\x00\x01+\x00\x04\xc7\xbf2I\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00" + dns_payload = b"\x009ib\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x08faceface\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00\x01\x00\x00\x01+\x00\x04\xc7\xbf2I\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00" http_payload = b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" value = value.replace(b"__DNS_REQUEST__", dns_payload) @@ -195,7 +194,17 @@ class Layer(): as a field properly. """ load = ''.join([random.choice(string.ascii_lowercase + string.digits) for k in range(10)]) - return urllib.parse.quote(load) + return random.choice(["", "__DNS_REQUEST__", "__HTTP_REQUEST__", urllib.parse.quote(load)]) + + +class RawLayer(Layer): + """ + Defines an interface for the scapy Raw layer. + """ + name = "Raw" + protocol = Raw + _fields = [] + fields = [] class IPLayer(Layer): @@ -273,7 +282,11 @@ class IPLayer(Layer): Sets the flags field. There is a bug in scapy, if you retrieve an empty flags field, it will return "", but you cannot set this value back. To reproduce this bug: + + .. code-block:: python + >>> setattr(IP(), "flags", str(IP().flags)) # raises a ValueError + To handle this case, this method converts empty string to zero so that it can be safely stored. """ @@ -400,9 +413,15 @@ class TCPLayer(Layer): 'dataofs' : self.gen_dataofs, 'flags' : self.gen_flags, 'chksum' : self.gen_chksum, - 'options' : self.gen_options + 'options' : self.gen_options, + 'window' : self.gen_window } + def gen_window(self, field): + """ + Generates a window size. + """ + return random.choice(range(10, 200, 10)) def gen_chksum(self, field): """ diff --git a/actions/packet.py b/actions/packet.py index f40b6d1..332768a 100644 --- a/actions/packet.py +++ b/actions/packet.py @@ -110,11 +110,10 @@ class Packet(): """ iter_packet = self.packet while iter_packet: - if iter_packet.name.lower() == "raw": - return parsed_layer = Packet.parse_layer(iter_packet) if parsed_layer: - yield parsed_layer + if parsed_layer.name != "Raw": + yield parsed_layer iter_packet = parsed_layer.get_next_layer() else: iter_packet = iter_packet.payload diff --git a/actions/sleep.py b/actions/sleep.py index bc7c3ae..8d2a059 100644 --- a/actions/sleep.py +++ b/actions/sleep.py @@ -1,8 +1,21 @@ from actions.action import Action + class SleepAction(Action): + """ + Defines the SleepAction - causes the engine to pause before sending a packet. + """ + # Do not select the sleep action during evolutions + frequency = 0 def __init__(self, time=1, environment_id=None): - Action.__init__(self, "sleep", "both") + """ + Initializes the sleep action. + + Args: + time (float): How much time the packet should delay before sending + environment_id (str, optional): Environment ID of the strategy this action is a part of + """ + Action.__init__(self, "sleep", "out") self.terminal = False self.branching = False self.time = time diff --git a/actions/sniffer.py b/actions/sniffer.py new file mode 100644 index 0000000..d4fc543 --- /dev/null +++ b/actions/sniffer.py @@ -0,0 +1,85 @@ +import threading +import os + +import actions.packet +from scapy.all import sniff +from scapy.utils import PcapWriter + + +class Sniffer(): + """ + The sniffer class lets the user begin and end sniffing whenever in a given location with a port to filter on. + Call start_sniffing to begin sniffing and stop_sniffing to stop sniffing. + """ + + def __init__(self, location, port, logger): + """ + Intializes a sniffer object. + Needs a location and a port to filter on. + """ + self.stop_sniffing_flag = False + self.location = location + self.port = port + self.pcap_thread = None + self.packet_dumper = None + self.logger = logger + full_path = os.path.dirname(location) + assert port, "Need to specify a port in order to launch a sniffer" + if not os.path.exists(full_path): + os.makedirs(full_path) + + def __packet_callback(self, scapy_packet): + """ + This callback is called whenever a packet is applied. + Returns true if it should finish, otherwise, returns false. + """ + packet = actions.packet.Packet(scapy_packet) + for proto in ["TCP", "UDP"]: + if(packet.haslayer(proto) and ((packet[proto].sport == self.port) or (packet[proto].dport == self.port))): + break + else: + return self.stop_sniffing_flag + + self.logger.debug(str(packet)) + self.packet_dumper.write(scapy_packet) + return self.stop_sniffing_flag + + def __spawn_sniffer(self): + """ + Saves pcaps to a file. Should be run as a thread. + Ends when the stop_sniffing_flag is set. Should not be called by user + """ + self.packet_dumper = PcapWriter(self.location, append=True, sync=True) + while(self.stop_sniffing_flag == False): + sniff(stop_filter=self.__packet_callback, timeout=1) + + def start_sniffing(self): + """ + Starts sniffing. Should be called by user. + """ + self.stop_sniffing_flag = False + self.pcap_thread = threading.Thread(target=self.__spawn_sniffer) + self.pcap_thread.start() + self.logger.debug("Sniffer starting to port %d" % self.port) + + def __enter__(self): + """ + Defines a context manager for this sniffer; simply starts sniffing. + """ + self.start_sniffing() + return self + + def __exit__(self, exc_type, exc_value, tb): + """ + Defines exit context manager behavior for this sniffer; simply stops sniffing. + """ + self.stop_sniffing() + + def stop_sniffing(self): + """ + Stops the sniffer by setting the flag and calling join + """ + if(self.pcap_thread): + self.stop_sniffing_flag = True + self.pcap_thread.join() + self.logger.debug("Sniffer stopping") diff --git a/actions/strategy.py b/actions/strategy.py index 613f029..0adb190 100644 --- a/actions/strategy.py +++ b/actions/strategy.py @@ -8,6 +8,8 @@ class Strategy(object): def __init__(self, in_actions, out_actions, environment_id=None): self.in_actions = in_actions self.out_actions = out_actions + self.descendents = [] + self.in_enabled = True self.out_enabled = True @@ -52,6 +54,48 @@ class Strategy(object): rep += "%s\n" % action_tree.pretty_print() return rep + def initialize(self, logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, disabled=None): + """ + Initializes a new strategy object randomly. + """ + # Disable specific forests if none requested + if num_in_trees == 0: + self.in_enabled = False + + if num_out_trees == 0: + self.out_enabled = False + + # If a specific population seed is requested, build using that + if seed: + starting_strat = actions.utils.parse(seed, logger) + self.out_actions = starting_strat.out_actions + self.in_actions = starting_strat.in_actions + return self + + self.init_from_scratch(num_in_trees, num_out_trees, num_in_actions, num_out_actions, disabled=disabled) + return self + + def init_from_scratch(self, num_in_trees, num_out_trees, num_in_actions, num_out_actions, disabled=None): + """ + Initializes this individual by drawing random actions. + """ + for _ in range(0, num_in_trees): + # Define a new in action tree + in_tree = actions.tree.ActionTree("in") + # Initialize the in tree + in_tree.initialize(num_in_actions, self.environment_id, disabled=disabled) + # Add them to this strategy + self.in_actions.append(in_tree) + + for _ in range(0, num_out_trees): + # Define a new out action tree + out_tree = actions.tree.ActionTree("out") + # Initialize the out tree + out_tree.initialize(num_out_actions, self.environment_id, disabled=disabled) + # Add them to this strategy + self.out_actions.append(out_tree) + + def act_on_packet(self, packet, logger, direction="out"): """ Runs the strategy on a given scapy packet. @@ -62,6 +106,7 @@ class Strategy(object): return [packet] return self.run_on_packet(packet, logger, direction) + def run_on_packet(self, packet, logger, direction): """ Runs the strategy on a given packet given the forest direction. @@ -86,3 +131,99 @@ class Strategy(object): if not ran: packets_to_send = [packet] return packets_to_send + + def mutate_dir(self, trees, direction, logger): + """ + Mutates a list of trees. Requires the direction the tree operates on + (in or out). + """ + pick = random.uniform(0, 1) + if pick < 0.1 or not trees: + new_tree = actions.tree.ActionTree(direction) + new_tree.initialize(1, self.environment_id) + trees.append(new_tree) + elif pick < 0.2 and trees: + trees.remove(random.choice(trees)) + elif pick < 0.25 and trees and len(trees) > 1: + random.shuffle(trees) + else: + for action_tree in trees: + action_tree.mutate() + + def mutate(self, logger): + """ + Top level mutation function for a strategy. Simply mutates the out + and in trees. + """ + if self.in_enabled: + self.mutate_dir(self.in_actions, "in", logger) + if self.out_enabled: + self.mutate_dir(self.out_actions, "out", logger) + return self + + +def swap_one(forest1, forest2): + """ + Swaps a random tree from forest1 and forest2. + + It picks a random element within forest1 and a random element within forest2, + chooses a random index within each forest, and inserts the random element + """ + assert type(forest1) == list + assert type(forest2) == list + rand_idx1, rand_idx2 = 0, 0 + donation, other_donation = None, None + if forest1: + donation = random.choice(forest1) + forest1.remove(donation) + if len(forest1) > 0: + rand_idx1 = random.choice(list(range(0, len(forest1)))) + + if forest2: + other_donation = random.choice(forest2) + forest2.remove(other_donation) + if len(forest2) > 0: + rand_idx2 = random.choice(list(range(0, len(forest2)))) + + if other_donation: + forest1.insert(rand_idx1, other_donation) + + if donation: + forest2.insert(rand_idx2, donation) + + return True + + +def do_mate(forest1, forest2): + """ + Performs mating between two given forests (lists of trees). + With 80% probability, a random tree from each forest are mated, + otherwise, a random tree is swapped between them. + """ + # If 80% and there are trees in both forests to mate, or + # if there is only 1 tree in each forest, mate those trees + if (random.random() < 0.8 and forest1 and forest2) or \ + (len(forest1) == 1 and len(forest2) == 1): + tree1 = random.choice(forest1) + tree2 = random.choice(forest2) + return tree1.mate(tree2) + # Otherwise, swap a random tree from each forest + elif forest1 or forest2: + return swap_one(forest1, forest2) + return False + + +def mate(ind1, ind2, indpb): + """ + Executes a uniform crossover that modify in place the two + individuals. The attributes are swapped according to the + *indpb* probability. + """ + out_success, in_success = True, True + if ind1.out_enabled and random.random() < indpb: + out_success = do_mate(ind1.out_actions, ind2.out_actions) + + if ind1.in_enabled and random.random() < indpb: + in_success = do_mate(ind1.in_actions, ind2.in_actions) + + return out_success and in_success diff --git a/actions/tamper.py b/actions/tamper.py index 480cb33..e9121a2 100644 --- a/actions/tamper.py +++ b/actions/tamper.py @@ -3,11 +3,11 @@ TamperAction One of the four packet-level primitives supported by Geneva. Responsible for any packet-level modifications (particularly header modifications). It supports the following primitives: - - no operation: it returns the packet given - - replace: it changes a packet field to a fixed value - - corrupt: it changes a packet field to a randomly generated value each time it is run - - add: adds a given value to the value in a field - - compress: performs DNS decompression on the packet (if applicable) +- no operation: it returns the packet given +- replace: it changes a packet field to a fixed value +- corrupt: it changes a packet field to a randomly generated value each time it is run +- add: adds a given value to the value in a field +- compress: performs DNS decompression on the packet (if applicable) """ from actions.action import Action @@ -20,18 +20,72 @@ import random # All supported tamper primitives SUPPORTED_PRIMITIVES = ["corrupt", "replace", "add", "compress"] +# Tamper primitives we can mutate to by default +ACTIVATED_PRIMITIVES = ["replace", "corrupt", "add"] + class TamperAction(Action): """ Defines the TamperAction for Geneva. """ + frequency = 5 def __init__(self, environment_id=None, field=None, tamper_type=None, tamper_value=None, tamper_proto="TCP"): + """ + Creates a tamper object. + + Args: + environment_id (str, optional): environment_id of a previously run strategy, used to find packet captures + field (str, optional): field that the object will tamper. If not set, all the parameters are chosen randomly + tamper_type (str, optional): primitive this tamper will use ("corrupt") + tamper_value (str, optional): value to tamper to + tamper_proto (str, optional): protocol we are tampering + """ Action.__init__(self, "tamper", "both") self.field = field self.tamper_value = tamper_value self.tamper_proto = actions.utils.string_to_protocol(tamper_proto) self.tamper_proto_str = tamper_proto + self.tamper_type = tamper_type + if not self.tamper_type: + self._mutate_tamper_type() + + if not self.field: + self._mutate(environment_id) + + def mutate(self, environment_id=None): + """ + Mutate can switch between the tamper type, field. + """ + # With some probability switch tamper types + pick = random.random() + if pick < 0.2: + self._mutate_tamper_type() + else: + self._mutate(environment_id) + + def _mutate_tamper_type(self): + """ + Randomly picks a tamper type to change to. + """ + self.tamper_type = random.choice(ACTIVATED_PRIMITIVES) + if self.tamper_type == "compress": + self.tamper_proto_str = "DNS" + self.tamper_proto = actions.utils.string_to_protocol(self.tamper_proto_str) + self.field = "qd" + + def _mutate(self, environment_id): + """ + Mutates this action using: + - previously seen packets with 50% probability + - a fuzzed packet with 50% probability + """ + # Retrieve a new protocol and field options for this protocol + proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, 0.5) + self.tamper_proto = proto + self.tamper_proto_str = proto.__name__ + self.field = field + self.tamper_value = value def tamper(self, packet, logger): """ diff --git a/actions/trace.py b/actions/trace.py index c1df0d7..01df65f 100644 --- a/actions/trace.py +++ b/actions/trace.py @@ -15,8 +15,19 @@ class TraceAction(Action): TraceAction is an experimental action that is never used in actual evolution """ + # Do not select Trace during evolutions + frequency = 0 def __init__(self, start_ttl=1, end_ttl=64, environment_id=None): + """ + Initializes the trace action. + + Args: + start_ttl (int): Starting TTL to use + end_ttl (int): TTL to end with + environment_id (str, optional): Environment ID associated with the strategy we are a part of + """ Action.__init__(self, "trace", "out") + self.enabled = True self.terminal = True self.branching = False self.start_ttl = start_ttl @@ -40,7 +51,7 @@ class TraceAction(Action): return packet, None if self.ran: - logger.debug(" - trace action already ran. Dropping given traffic. To reset this action, restart the engine.") + logger.debug(" - trace action already ran. Dropping given traffic.") return None, None self.ran = True diff --git a/actions/tree.py b/actions/tree.py index c757bde..c083562 100644 --- a/actions/tree.py +++ b/actions/tree.py @@ -24,12 +24,35 @@ class ActionTree(): """ def __init__(self, direction, trigger=None): + """ + Creates this action tree. + + Args: + direction (str): Direction this tree is facing ("out", "in") + trigger (:obj:`actions.trigger.Trigger`): Trigger to use with this tree + """ self.trigger = trigger self.action_root = None self.direction = direction self.environment_id = None self.ran = False + def initialize(self, num_actions, environment_id, allow_terminal=True, disabled=None): + """ + Sets up this action tree with a given number of random actions. + Note that the returned action trees may have less actions than num_actions + if terminal actions are used. + """ + self.environment_id = environment_id + self.trigger = actions.trigger.Trigger(None, None, None, environment_id=environment_id) + if not allow_terminal or random.random() > 0.1: + allow_terminal = False + + for _ in range(num_actions): + new_action = self.get_rand_action(self.direction, disabled=disabled) + self.add_action(new_action) + return self + def __iter__(self): """ Sets up a preoder iterator for the tree. @@ -222,8 +245,8 @@ class ActionTree(): if not node.right: yield right_packet - # If we have a left action and were given a packet to pass on, run - # on the left packet + # If we have a right action and were given a packet to pass on, run + # on the right packet if node.right and right_packet: for rpacket in self.do_run(node.right, right_packet, logger): yield rpacket @@ -383,6 +406,22 @@ class ActionTree(): break return action_added + def get_rand_action(self, direction, request=None, allow_terminal=True, disabled=None): + """ + Retrieves and initializes a random action that can run in the given direction. + """ + pick = random.random() + action_options = actions.action.Action.get_actions(direction, disabled=disabled, allow_terminal=allow_terminal) + # Check to make sure there are still actions available to use + assert action_options, "No actions were available" + act_dict = {} + all_opts = [] + for action_name, act_cls in action_options: + act_dict[action_name] = act_cls + all_opts += ([act_cls] * act_cls.frequency) + new_action = act_dict.get(request, random.choice(all_opts)) + return new_action(environment_id=self.environment_id) + def remove_one(self): """ Removes a random leaf from the tree. @@ -392,6 +431,26 @@ class ActionTree(): action = random.choice(self) return self.remove_action(action) + def mutate(self): + """ + Mutates this action tree with respect to a given direction. + """ + pick = random.uniform(0, 1) + if pick < 0.20 or not self.action_root: + new_action = self.get_rand_action(direction=self.direction) + self.add_action(new_action) + elif pick < 0.65 and self.action_root: + action = random.choice(self) + action.mutate(environment_id=self.environment_id) + # If this individual has never been run under the evaluator, + # or if it ran and it failed, it won't have an environment_id, + # which means it has no saved packets to read from. + elif pick < 0.80 and self.environment_id: + self.trigger.mutate(self.environment_id) + else: + self.remove_one() + return self + def choose_one(self): """ Picks a random element in the tree. @@ -415,6 +474,40 @@ class ActionTree(): return action, "right" return None, None + def swap(self, my_donation, other_tree, other_donation): + """ + Swaps a node in this tree with a node in another tree. + """ + parent, direction = self.get_parent(my_donation) + other_parent, other_direction = other_tree.get_parent(other_donation) + # If this tree is empty or I'm trying to donate my root + if not my_donation or not parent: + parent = self + direction = "action_root" + # if the other tree is empty or they are trying to donate their root + if not other_donation or not other_parent: + other_parent = other_tree + other_direction = "action_root" + + setattr(parent, direction, other_donation) + setattr(other_parent, other_direction, my_donation) + + return True + + def mate(self, other_tree): + """ + Mates this tree with another tree. + """ + # If both trees are empty, nothing to do + if not self.action_root and not other_tree.action_root: + return False + + # Chose an action node in this tree to swap + my_swap_node = self.choose_one() + other_swap_node = other_tree.choose_one() + + return self.swap(my_swap_node, other_tree, other_swap_node) + def pretty_print_help(self, root, visual=False, parent=None): """ Pretty prints the tree. @@ -448,6 +541,7 @@ class ActionTree(): return newroot + def pretty_print(self, visual=False): """ Pretty prints the tree. diff --git a/actions/trigger.py b/actions/trigger.py index 5815b6d..3c97574 100644 --- a/actions/trigger.py +++ b/actions/trigger.py @@ -15,6 +15,7 @@ class Trigger(object): - trigger_value: the value in the trigger_field that, upon a match, will cause the trigger to fire - environment_id: environment_id the current trigger is running under. Used to retrieve previously saved packets - gas: how many times this trigger can fire before it stops triggering. gas=None disables gas (unlimited triggers.) + - has_wildcard: represents if the trigger will match a specific value, or any value containing trigger_value """ self.trigger_type = trigger_type self.trigger_field = trigger_field @@ -23,9 +24,55 @@ class Trigger(object): self.environment_id = environment_id self.num_seen = 0 self.gas_remaining = gas + self.has_wildcard = False # Bomb triggers act like reverse triggers. They run the action only after the action has been triggered x times self.bomb_trigger = bool(gas and gas < 0) self.ran = False + # ignore numerical trigger values + if isinstance(self.trigger_value, (str)): + # check if value field is wildcarded or not + if(len(self.trigger_value) != 0 and self.trigger_value[-1] == '*'): + self.has_wildcard = True + # remove '*' wildcard from trigger_value for ease of use + self.trigger_value = self.trigger_value[:-1] + if not self.trigger_type: + self.trigger_type, self.trigger_proto, self.trigger_field, self.trigger_value, self.gas_remaining = Trigger.get_rand_trigger(environment_id, 1) + + @staticmethod + def get_gas(): + """ + Returns a random value for gas for this trigger. + """ + if GAS_ENABLED and random.random() < 0.2: + # Use gas in 20% of scenarios + # Pick a number for gas between 0 - 5 + gas_remaining = int(random.random() * 5) + else: + # Do not use gas + gas_remaining = None + return gas_remaining + + @staticmethod + def get_rand_trigger(environment_id, real_packet_probability): + """ + Creates a random trigger. + """ + proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, real_packet_probability, enable_options=False, enable_load=False) + gas_remaining = Trigger.get_gas() + if not FIXED_TRIGGER: + # Only "field" triggers are supported currently + return "field", proto.__name__, field, value, gas_remaining + return (FIXED_TRIGGER.trigger_type, + FIXED_TRIGGER.trigger_proto, + FIXED_TRIGGER.trigger_field, + FIXED_TRIGGER.trigger_value, + FIXED_TRIGGER.gas_remaining) + + def mutate(self, environment_id, real_packet_probability=0.5): + """ + Mutates this trigger object by picking a new protocol, field, and value. + """ + self.trigger_type, self.trigger_proto, self.trigger_field, self.trigger_value, self.gas_remaining = Trigger.get_rand_trigger(environment_id, real_packet_probability) def is_applicable(self, packet, logger): """ @@ -37,7 +84,10 @@ class Trigger(object): return False packet_value = packet.get(self.trigger_proto, self.trigger_field) - will_run = (self.trigger_value == packet_value) + if self.has_wildcard: + will_run = (self.trigger_value in packet_value) + else: + will_run = (self.trigger_value == packet_value) # Track if this action is used if (not GAS_ENABLED or self.gas_remaining is None) and will_run: diff --git a/actions/utils.py b/actions/utils.py index 61db62b..d643e95 100644 --- a/actions/utils.py +++ b/actions/utils.py @@ -2,6 +2,7 @@ import copy import datetime import importlib import inspect +import json import logging import os import string @@ -12,24 +13,38 @@ import urllib.parse import actions.action import actions.trigger import actions.packet +import plugins.plugin_client +import plugins.plugin_server from scapy.all import TCP, IP, UDP, rdpcap import netifaces -RUN_DIRECTORY = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") +RUN_DIRECTORY = os.path.join("trials", datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")) # Hard coded options FLAGFOLDER = "flags" # Holds copy of console file handler's log level -CONSOLE_LOG_LEVEL = logging.DEBUG +CONSOLE_LOG_LEVEL = "debug" BASEPATH = os.path.dirname(os.path.abspath(__file__)) PROJECT_ROOT = os.path.dirname(BASEPATH) +class SkipStrategyException(Exception): + """ + Raised to signal that this strategy evaluation should be cut off. + """ + def __init__(self, msg, fitness): + """ + Creates the exception with the fitness to pass back + """ + self.fitness = fitness + self.msg = msg + + def parse(requested_trees, logger): """ Parses a string representation of a solution into its object form. @@ -71,7 +86,9 @@ def parse(requested_trees, logger): # strategies, so restore the "|" that was lost from the split str_action = str_action + "|" new_tree = actions.tree.ActionTree(direction) - new_tree.parse(str_action, logger) + success = new_tree.parse(str_action, logger) + if success is False: + raise actions.tree.ActionTreeParseError("Failed to parse tree") # Once all the actions are parsed, add this tree to the # current direction of actions @@ -85,7 +102,7 @@ def parse(requested_trees, logger): return strat -def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_level=logging.DEBUG): +def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_level="DEBUG"): """ Configures and returns a logger. """ @@ -100,7 +117,7 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev os.makedirs(flag_path) # Set up a client logger logger = logging.getLogger(logger_name + environment_id) - logger.setLevel(logging.DEBUG) + logger.setLevel("DEBUG") # Disable the root logger to avoid double printing logger.propagate = False @@ -108,7 +125,7 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev if logger.handlers: return logger fh = logging.FileHandler(os.path.join(basepath, log_dir, "logs", "%s.%s.log" % (environment_id, log_name))) - fh.setLevel(logging.DEBUG) + fh.setLevel("DEBUG") log_prefix = "[%s] " % log_name.upper() formatter = logging.Formatter("%(asctime)s %(levelname)s:" + log_prefix + "%(message)s", datefmt="%Y-%m-%d %H:%M:%S") @@ -119,7 +136,7 @@ def get_logger(basepath, log_dir, logger_name, log_name, environment_id, log_lev ch = logging.StreamHandler() ch.setFormatter(formatter) ch.setLevel(log_level) - CONSOLE_LOG_LEVEL = ch.level + CONSOLE_LOG_LEVEL = log_level.lower() logger.addHandler(ch) return logger @@ -135,6 +152,34 @@ def close_logger(logger): handler.close() +class Logger(): + """ + Logging class context manager, as a thin wrapper around the logging class to help + handle closing open file descriptors. + """ + def __init__(self, log_dir, logger_name, log_name, environment_id, log_level="DEBUG"): + self.log_dir = log_dir + self.logger_name = logger_name + self.log_name = log_name + self.environment_id = environment_id + self.log_level = log_level + self.logger = None + + def __enter__(self): + """ + Sets up a logger. + """ + self.logger = get_logger(PROJECT_ROOT, self.log_dir, self.logger_name, self.log_name, self.environment_id, log_level=self.log_level) + return self.logger + + def __exit__(self, exc_type, exc_value, tb): + """ + Closes file handles. + """ + close_logger(self.logger) + + + def get_console_log_level(): """ returns log level of console handler @@ -142,6 +187,84 @@ def get_console_log_level(): return CONSOLE_LOG_LEVEL +def get_plugins(): + """ + Iterates over this current directory to retrieve plugins. + """ + plugins = [] + for f in os.listdir(os.path.join(PROJECT_ROOT, "plugins")): + if os.path.isdir(os.path.join(PROJECT_ROOT, "plugins", f)) and "__pycache__" not in f: + plugins.append(f) + return plugins + + +def import_plugin(plugin, side): + """ + Imports given plugin. + Args: + - plugin: plugin to import (e.g. "http") + - side: which side of the connection should be imported ("client" or "server") + """ + + # Define the full module for this plugin + mod = "plugins.%s.%s" % (plugin, side) + + path = os.path.join(PROJECT_ROOT, "plugins", plugin) + if path not in sys.path: + sys.path.append(path) + + # Import the module + importlib.import_module(mod) + + # Predicate to filter classmembers + def check_plugin(obj): + """ + Filters class members to ensure we get only enabled Plugin subclasses + """ + return inspect.isclass(obj) and \ + issubclass(obj, plugins.plugin.Plugin) and \ + (obj != plugins.plugin_client.ClientPlugin and \ + obj != plugins.plugin_server.ServerPlugin and \ + obj != plugins.plugin.Plugin) and \ + obj(None).enabled + + # Filter the class members of the imported module to find our Plugin subclass + clsmembers = inspect.getmembers(sys.modules[mod], predicate=check_plugin) + + # Sanity check the class members we identified + assert clsmembers, "Could not find plugin %s" % mod + assert len(clsmembers) == 1, "Too many matching plugins found for %s" % mod + + # Extract the class - clsmembers[0] is a tuple of (name, class) + _, cls = clsmembers[0] + + # Return the module path and class + return mod, cls + + +def build_command(args): + """ + Given a dictionary of arguments, build it back into a command line string. + """ + cmd = [] + for opt in args: + # Don't pass along store true args that are false + if args[opt] in [False, None]: + continue + cmd.append("--%s" % opt.replace("_", "-")) + # If store true arg, we don't need to pass the value + if args[opt] is True: + continue + + if args[opt] is '': + cmd.append("''") + elif " " in str(args[opt]): + cmd.append("\"" + str(args[opt]) + "\"") + else: + cmd.append(str(args[opt])) + return cmd + + def string_to_protocol(protocol): """ Converts string representations of scapy protocol objects to @@ -177,6 +300,103 @@ def setup_dirs(output_dir): return ga_log_dir +def get_from_fuzzed_or_real_packet(environment_id, real_packet_probability, enable_options=True, enable_load=True): + """ + Retrieves a protocol, field, and value from a fuzzed or real packet, depending on + the given probability and if given packets is not None. + """ + packets = actions.utils.read_packets(environment_id) + if packets and random.random() < real_packet_probability: + packet = random.choice(packets) + return packet.get_random() + return actions.packet.Packet().gen_random() + + +def read_packets(environment_id): + """ + Reads the pcap file associated with the last evaluation of this strategy. + Returns a list of Geneva Packet objects. + """ + if not environment_id: + return None + + packets_path = os.path.join(RUN_DIRECTORY, "packets", "original_" + str(environment_id) + ".pcap") + if not os.path.exists(packets_path): + return None + + parsed = [] + try: + packets = rdpcap(packets_path) + parsed = [actions.packet.Packet(p) for p in packets] + except Exception as e: + print(e) + print("FAILED TO PARSE!") + + return parsed + + +def punish_fitness(fitness, logger, eng): + """ + Adjusts fitness based on additional optimizer functions. + """ + if not eng: + logger.warning("Requested fitness adjustment without an engine - returning original fitness.") + return fitness + logger.debug("Initiating fitness adjustment") + if eng and eng.strategy: + fitness = punish_complexity(fitness, logger, eng.strategy) + fitness = punish_unused(fitness, logger, eng.strategy) + if fitness > 0: + overhead = int(eng.overhead / 2) + logger.debug("Punishing for overhead: %d" % overhead) + fitness -= overhead + + return fitness + + +def punish_unused(fitness, logger, ind): + """ + Punishes strategy for each action that was not run. + """ + if not ind: + return fitness + logger.debug("Punishing for unused actions") + num_unused = [action_tree.ran for action_tree in ind.out_actions].count(False) + fitness -= (num_unused * 10) + logger.debug(" - Number of unused actions in out forest: %d" % num_unused) + num_unused = [action_tree.ran for action_tree in ind.in_actions].count(False) + fitness -= (num_unused * 10) + logger.debug(" - Number of unused actions in in forest: %d" % num_unused) + return fitness + + +def punish_complexity(fitness, logger, ind): + """ + Reduces fitness based on number of actions - optimizes for simplicity. + """ + if not ind: + return fitness + # Punish for number of actions + if fitness > 0: + logger.debug("Punishing for complexity: %d" % len(ind)) + fitness -= len(ind) + return fitness + + +def write_fitness(fitness, output_path, eid): + """ + Writes fitness to disk. + """ + try: + float(fitness) + except ValueError: + print("Given fitness (%r) is not a number!" % fitness) + raise + fitpath = os.path.join(PROJECT_ROOT, output_path, FLAGFOLDER, eid) + ".fitness" + with open(fitpath, "w") as fitfile: + fitfile.write(str(fitness)) + + def get_interface(): """ Chooses an interface on the machine to use for socket testing. @@ -189,3 +409,26 @@ def get_interface(): # Filter for IPv4 addresses if netifaces.AF_INET in info: return iface + + +def get_worker(name, logger): + """ + Returns information dictionary about a worker given its name. + """ + path = os.path.join("workers", name, "worker.json") + if os.path.exists(name): + path = name + + dirpath = os.path.dirname(path) + + if not os.path.exists(path): + return None + + with open(path, "r") as fd: + data = json.load(fd) + + # If there is a private key, update the path to be relative to the project base + if data.get("keyfile"): + data["keyfile"] = os.path.join(dirpath, data["keyfile"]) + + return data diff --git a/censors/censor.py b/censors/censor.py new file mode 100644 index 0000000..f5724f7 --- /dev/null +++ b/censors/censor.py @@ -0,0 +1,163 @@ +import socket +socket.setdefaulttimeout(1) +import logging +import random +import os + +import actions.packet +import actions.utils + +# Squelch annoying scapy ::1 runtime errors +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + +# Netfilterqueue may not work outside the docker container, +# but this file can still be imported outside the docker container +try: + from netfilterqueue import NetfilterQueue +except ImportError: + pass + +from scapy.all import send, IP + +# Note that censor.py lives in censors, so we need an extra dirname() call to get +# to the project root +BASEPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class Censor(object): + def __init__(self, eid, log_dir, log_level, port, queue_num): + """ + Setup censor attributes and logging. + """ + self.enabled = True + self.nfqueue = None + self.running_nfqueue = False + self.queue_num = queue_num + self.port = port + self.eid = eid + self.logger = None + self.log_dir = log_dir + if log_level: + self.logger = actions.utils.get_logger(BASEPATH, log_dir, __name__, "censor", eid, log_level=log_level) + self.logger.debug("Censor created to port %d on queue %d" % (port, queue_num)) + + def start(self): + """ + Initialize the censor. + """ + self.logger.debug("Censor initializing.") + + # Set up iptables rules to catch packets + os.system("iptables -A FORWARD -j NFQUEUE -p tcp --sport %s --queue-num %s" % (self.port, self.queue_num)) + os.system("iptables -A FORWARD -j NFQUEUE -p tcp --dport %s --queue-num %s" % (self.port, self.queue_num)) + self.logger.debug("Censor iptables added") + + #self.running_nfqueue = True + self.num = 0 + try: + self.nfqueue = NetfilterQueue() + self.logger.debug("Censor binding") + self.nfqueue.bind(int(self.queue_num), self.callback) + self.logger.debug("Censor bound") + self.nfqueue.run() + except KeyboardInterrupt: + self.logger.debug("CENSOR GOT SHUTDOWN") + self.shutdown() + #self.nfqueue_socket = socket.fromfd(self.nfqueue.get_fd(), socket.AF_UNIX, socket.SOCK_STREAM) + #self.nfqueue_thread = threading.Thread(target=self.run_nfqueue) + #self.nfqueue_thread.start() + # Spin wait the main thread while running nfqueue in the other threads + #while self.running_nfqueue: + # time.sleep(1) + + def check_exit(self): + """ + Check if a shutdown flag has been written. + """ + flag_folder = os.path.join(BASEPATH, self.log_dir, actions.utils.FLAGFOLDER) + if not os.path.exists(flag_folder): + os.makedirs(flag_folder) + return os.path.exists(os.path.join(flag_folder, "shutdown")) + + def run_nfqueue(self): + """ + Run nfqueue in a non-blocking way. Note that nfqueue reports + that it supports non-blocking operation, but this is broken in the + library, and the following is the workaround. + """ + try: + while self.running_nfqueue: + try: + self.nfqueue.run_socket(self.nfqueue_socket) + except socket.timeout: + self.logger.debug("Exiting") + # Check if we need to exit + if self.check_exit(): + break + pass + self.shutdown() + except Exception: + self.logger.exception("Exception out of run_nfqueue()") + + def mysend(self, packet): + """ + Sends a packet with scapy. + """ + if "TCP" in packet: + self.logger.debug(actions.packet.Packet._str_packet(packet)) + send(packet, verbose=False) + return + + def get_payload(self, packet): + """ + Parse paylaod out of the given scapy packet. + """ + payload = bytes(packet["TCP"].payload) + if str(payload) != "b''": + return payload + else: + return b"" + + def shutdown(self): + """ + Shuts down and cleans up the censor. + """ + self.logger.debug("Shutting down censor.") + self.running_nfqueue = False + self.nfqueue.unbind() + #self.nfqueue_socket.close() + os.system("iptables -D FORWARD -j NFQUEUE -p tcp --sport %s --queue-num %s" % (self.port, self.queue_num)) + os.system("iptables -D FORWARD -j NFQUEUE -p tcp --dport %s --queue-num %s" % (self.port, self.queue_num)) + + def callback(self, packet): + """ + NFQueue bound callback to capture packets and check whether we + want to censor it. + """ + try: + scapy_packet = IP(packet.get_payload()) + # Check for control check packet from evaluator to announce readiness + if scapy_packet.sport == 2222 and scapy_packet.seq == 13337: + # This line cannot be removed - it is to signal to the client the censor is ready + flag_folder = os.path.join(BASEPATH, self.log_dir, actions.utils.FLAGFOLDER) + if not os.path.exists(flag_folder): + os.makedirs(flag_folder) + ready_path = os.path.join(flag_folder, "%s.censor_ready" % self.eid) + self.logger.debug("Writing ready file to %s" % ready_path) + if not os.path.exists(ready_path): + os.system("touch %s" % ready_path) + self.logger.debug("Censor ready.") + packet.drop() + return + action = "accept" + # Check if the packet should be censored + if self.check_censor(scapy_packet): + # If so, trigger the censoring itself (drop packet, send RST, etc) + action = self.censor(scapy_packet) + + if action == "drop": + packet.drop() + else: + packet.accept() + except Exception: + self.logger.exception("Censor exception in nfqueue callback.") diff --git a/censors/censor1.py b/censors/censor1.py new file mode 100644 index 0000000..8c444ef --- /dev/null +++ b/censors/censor1.py @@ -0,0 +1,86 @@ +""" +Censor 1 + +Designed to be run by the evaluator. + +TCP Censor that synchronizes on first SYN only, works 100% of the time, +drops all packets after a TCP forbidden keyword is detected. +""" + +import logging +import actions.packet +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor1(Censor): + """ + TCP Censor that synchronizes on first SYN only, works 100% of the time, + drops all packets after a TCP forbidden keyword is detected. + """ + + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = 0 + self.drop_all_from = None + self.num = 0 + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Initial TCP synchronization + if packet["TCP"].sprintf('%TCP.flags%') == "S": + self.tcb = packet["TCP"].seq + 1 + self.logger.debug(("Synchronizing TCB (%d) on packet " + actions.packet.Packet._str_packet(packet)) % self.tcb) + return False + + # If we're tracking this packet stream + if packet["TCP"].seq == self.tcb: + self.tcb += len(self.get_payload(packet)) + else: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Censor 1 exception caught") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" + + + + + + + + + + + diff --git a/censors/censor10.py b/censors/censor10.py new file mode 100644 index 0000000..8f4971a --- /dev/null +++ b/censors/censor10.py @@ -0,0 +1,176 @@ +""" +Censor 10 is a RST censor designed to more closely mimic GFW behavior. It +tracks multiple connections using TCBs, and will enter a TCB resynchronization +state if a RST or FIN is sent and the full tuple of the TCB matches (src, dst, +sport, dport) an existing TCB. It creates new TCBs for connections it is not +yet aware of, and it checks all checksums of incoming packets (and ignores those +that are incorrect), meaning insertion packets with incorrect checksums will not +work. +""" + +import netifaces +import actions.packet +from censors.censor import Censor +from scapy.all import raw, IP, TCP + + +class Censor10(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcbs = [] + self.flagged_ips = [] + self.resynchronize = {} + self.censor_interfaces = netifaces.interfaces() + if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces: + self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr'] + + + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. + Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: %s" % actions.packet.Packet._str_packet(packet)) + if packet["IP"].src in self.flagged_ips: + self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Record the reported checksum for the incoming packet + reported_chksum = packet["TCP"].chksum + # Remove the checksum for the packet so we can recalculate it + del packet["TCP"].chksum + + # Note this is actually what scapy's show2 method does under the hood + # if curious, (see packet.py in scapy for show2 details) + calculated_chksum = packet.__class__(raw(packet))["TCP"].chksum + if reported_chksum != calculated_chksum: + self.logger.debug("Packet checksum (%d) is incorrect (correct=%d). Ignoring." % (reported_chksum, calculated_chksum)) + return False + + # If we are in a resynchronization state, or we do not yet have a connection and a new one + # is being created, add or update a TCB + tcb = self.get_matching_tcb(packet) + if (tcb and self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])]) or \ + (not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S"]): + + # Check if we've been tracking a connection for this ip:port <-> ip:port already, + # so we can just replace that tcb with updated info + if not tcb: + tcb = self.get_partial_tcb(packet) + if tcb is None: + self.logger.debug("Making a new TCB for packet %s" % actions.packet.Packet._str_packet(packet)) + tcb = {} + + tcb["src"] = packet["IP"].src + tcb["dst"] = packet["IP"].dst + tcb["sport"] = packet["TCP"].sport + tcb["dport"] = packet["TCP"].dport + tcb["seq"] = packet["TCP"].seq + # If we're synchronizing on a SYN flag, need to add 1. + if packet["TCP"].sprintf('%TCP.flags%') in ["S"]: + tcb["seq"] += 1 + else: + tcb["seq"] += len(self.get_payload(packet)) + + self.tcbs.append(tcb) + self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = False + self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), actions.packet.Packet._str_packet(packet))) + return False + + # If connection is getting torn down + elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "F"]: + self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = True + self.logger.debug(("Entering resynchronization state on packet %s" % actions.packet.Packet._str_packet(packet))) + + if not tcb: + self.logger.debug("No TCB matches packet.") + return False + + # Keep the TCB up to date + tcb["seq"] += len(self.get_payload(packet)) + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: %s" % actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 10") + return False + + def censor(self, scapy_packet): + """ + Adds client and server IPs to flagged IP list. + """ + if scapy_packet["IP"].src not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].src) + self.logger.debug("Marking IP %s for censorship..." % scapy_packet["IP"].src) + if scapy_packet["IP"].dst not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].dst) + self.logger.debug("Marking IP %s for censorship..." % scapy_packet["IP"].dst) + + client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src) + client_tcp_rst = TCP( + dport=scapy_packet[TCP].sport, + sport=scapy_packet[TCP].dport, + ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), + seq=scapy_packet[TCP].ack, + flags="R" + ) + client_rst = client_ip_rst / client_tcp_rst + + server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst) + server_tcp_rst = TCP( + dport=scapy_packet[TCP].dport, + sport=scapy_packet[TCP].sport, + ack=scapy_packet[TCP].ack, + seq=scapy_packet[TCP].seq, + flags="R" + ) + server_tcp_rst.show() + server_rst = server_ip_rst / server_tcp_rst + + for _ in range(0, 5): + self.mysend(client_rst) + self.mysend(server_rst) + + return "accept" + + def get_matching_tcb(self, packet): + """ + Checks if the packet matches the stored TCB. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"] and \ + packet["TCP"].seq == tcb["seq"]): + return tcb + return None + + def get_partial_tcb(self, packet): + """ + Checks if the packet matches an existing connection, regardless if the SEQ/ACK + are correct. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"]): + return tcb + return None diff --git a/censors/censor11.py b/censors/censor11.py new file mode 100644 index 0000000..4512f52 --- /dev/null +++ b/censors/censor11.py @@ -0,0 +1,176 @@ +""" +Censor 11 is a RST censor designed to more closely mimic GFW behavior. It +tracks multiple connections using TCBs, and will enter a TCB resynchronization +state if a RST or FIN is sent and the full tuple of the TCB matches (src, dst, +sport, dport) an existing TCB. It creates new TCBs for connections it is not +yet aware of, and it checks all checksums of incoming packets (and ignores those +that are incorrect), meaning insertion packets with incorrect checksums will not +work. It also resynchronizes on both S and ACK, defeating strategies that trigger +before the 3-way handshake has finished. +""" + +import actions.packet +import netifaces +from censors.censor import Censor +from scapy.all import raw, IP, TCP + + +class Censor11(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcbs = [] + self.flagged_ips = [] + self.resynchronize = {} + self.censor_interfaces = netifaces.interfaces() + if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces: + self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr'] + + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. + Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: %s" % actions.packet.Packet._str_packet(packet)) + if packet["IP"].src in self.flagged_ips: + self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + if packet["TCP"].dataofs < 5: + return False + # Record the reported checksum for the incoming packet + reported_chksum = packet["TCP"].chksum + # Remove the checksum for the packet so we can recalculate it + del packet["TCP"].chksum + + # Note this is actually what scapy's show2 method does under the hood + # if curious, (see packet.py in scapy for show2 details) + calculated_chksum = packet.__class__(raw(packet))["TCP"].chksum + if reported_chksum != calculated_chksum: + self.logger.debug("Packet checksum (%d) is incorrect (correct=%d). Ignoring." % (reported_chksum, calculated_chksum)) + return False + + # If we are in a resynchronization state, or we do not yet have a connection and a new one + # is being created, add or update a TCB + tcb = self.get_matching_tcb(packet) + if (tcb and self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])]) or \ + (not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S", "A"]): + + # Check if we've been tracking a connection for this ip:port <-> ip:port already, + # so we can just replace that tcb with updated info + tcb = self.get_partial_tcb(packet) + if tcb is None: + self.logger.debug("Making a new TCB for packet %s" % actions.packet.Packet._str_packet(packet)) + tcb = {} + + tcb["src"] = packet["IP"].src + tcb["dst"] = packet["IP"].dst + tcb["sport"] = packet["TCP"].sport + tcb["dport"] = packet["TCP"].dport + tcb["seq"] = packet["TCP"].seq + # If we're synchronizing on a SYN flag, need to add 1. + if packet["TCP"].sprintf('%TCP.flags%') in ["S"]: + tcb["seq"] += 1 + else: + tcb["seq"] += len(self.get_payload(packet)) + + self.tcbs.append(tcb) + self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = False + self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), actions.packet.Packet._str_packet(packet))) + return False + + # If connection is getting torn down + elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "F", "RA", "FA"]: + self.resynchronize[(tcb["src"], tcb["dst"], tcb["sport"], tcb["dport"])] = True + self.logger.debug(("Entering resynchronization state on packet %s" % actions.packet.Packet._str_packet(packet))) + + if not tcb: + self.logger.debug("No TCB matches packet.") + return False + + # Keep the TCB up to date + tcb["seq"] += len(self.get_payload(packet)) + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: %s" % actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 10") + return False + + def censor(self, scapy_packet): + """ + Adds client and server IPs to flagged IP list. + """ + if scapy_packet["IP"].src not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].src) + self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].src) + if scapy_packet["IP"].dst not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].dst) + self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].dst) + + client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src) + client_tcp_rst = TCP( + dport=scapy_packet[TCP].sport, + sport=scapy_packet[TCP].dport, + ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), + seq=scapy_packet[TCP].ack, + flags="R" + ) + client_rst = client_ip_rst / client_tcp_rst + + server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst) + server_tcp_rst = TCP( + dport=scapy_packet[TCP].dport, + sport=scapy_packet[TCP].sport, + ack=scapy_packet[TCP].ack, + seq=scapy_packet[TCP].seq, + flags="R" + ) + server_tcp_rst.show() + server_rst = server_ip_rst / server_tcp_rst + + for _ in range(0, 5): + self.mysend(client_rst) + self.mysend(server_rst) + + return "accept" + + def get_matching_tcb(self, packet): + """ + Checks if the packet matches the stored TCB. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"] and \ + packet["TCP"].seq == tcb["seq"]): + return tcb + return None + + def get_partial_tcb(self, packet): + """ + Checks if the packet matches an existing connection, regardless if the SEQ/ACK + are correct. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"]): + return tcb + return None diff --git a/censors/censor2.py b/censors/censor2.py new file mode 100644 index 0000000..a60466b --- /dev/null +++ b/censors/censor2.py @@ -0,0 +1,63 @@ +""" +Censor 2 ----> CENSOR 1 + +Designed to be run by the evaluator. + +TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to client. +""" + +import actions.packet +import logging +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor2(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = 0 + self.drop_all_from = None + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: %s", actions.packet.Packet._str_packet(packet)) + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + if packet["TCP"].sprintf('%TCP.flags%') == "S": + self.tcb = packet["TCP"].seq + 1 + self.logger.debug("Synchronizing TCB on packet: %s", actions.packet.Packet._str_packet(packet)) + return False + + if packet["TCP"].seq == self.tcb: + self.tcb += len(self.get_payload(packet)) + + else: + self.logger.debug("Ignoring packet: %s", actions.packet.Packet._str_packet(packet)) + return False + + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: %s", actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Censor 2 exception caught.") + return False + + def censor(self, scapy_packet): + """ + Send 5 resets to the client. + """ + rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src)/TCP(dport=scapy_packet[TCP].sport, sport=scapy_packet[TCP].dport, ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), seq=scapy_packet[TCP].ack, flags="R") + for i in range(0, 5): + self.mysend(rst) diff --git a/censors/censor3.py b/censors/censor3.py new file mode 100644 index 0000000..8aa5d7b --- /dev/null +++ b/censors/censor3.py @@ -0,0 +1,98 @@ +""" +Censor 3 ----> CENSOR 2 + +Designed to be run by the evaluator. + +TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to +server AND client. +""" + +import logging +import netifaces +import actions.packet +# Disable scapy ::1 warnings +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + +from scapy.all import IP, TCP, wrpcap + +from censors.censor import Censor + + +class Censor3(Censor): + """ + TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to + server AND client. + """ + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.enabled = True + self.tcb = 0 + self.drop_all_from = None + self.num = 0 + self.censor_interfaces = netifaces.interfaces() + if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces: + self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr'] + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.num += 1 + + # Only censor TCP packets for now + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if "TCP" not in packet: + return False + + if packet["TCP"].sprintf('%TCP.flags%') == "S": + self.tcb = packet["TCP"].seq + 1 + self.logger.debug("Synchronizing TCB on packet " + actions.packet.Packet._str_packet(packet)) + return False + + if packet["TCP"].seq == self.tcb: + self.tcb += len(self.get_payload(packet)) + + else: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Censor 3 Error caught.") + return False + + def censor(self, scapy_packet): + """ + Send 5 resets to the client and the server. + """ + client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src) + client_tcp_rst = TCP( + dport=scapy_packet[TCP].sport, + sport=scapy_packet[TCP].dport, + ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), + seq=scapy_packet[TCP].ack, + flags="R" + ) + client_rst = client_ip_rst / client_tcp_rst + + server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst) + server_tcp_rst = TCP( + dport=scapy_packet[TCP].dport, + sport=scapy_packet[TCP].sport, + ack=scapy_packet[TCP].ack, + seq=scapy_packet[TCP].seq, + flags="R" + ) + server_tcp_rst.show() + server_rst = server_ip_rst / server_tcp_rst + + for _ in range(0, 5): + self.mysend(client_rst) + self.mysend(server_rst) diff --git a/censors/censor4.py b/censors/censor4.py new file mode 100644 index 0000000..760cf11 --- /dev/null +++ b/censors/censor4.py @@ -0,0 +1,86 @@ +""" +Censor 4 + +Dropping censor that synchronizes TCB on all SYN and ACK packets. +""" + +import actions.packet +import logging +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor4(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = 0 + self.drop_all_from = None + self.num = 0 + + def check_censor(self, packet, verbose=False): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.num += 1 + + + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Initial TCP synchronization + if "S" == packet["TCP"].sprintf('%TCP.flags%'): + self.tcb = packet["TCP"].seq + 1 + self.logger.debug(("Synchronizing TCB (%d) on S packet " + actions.packet.Packet._str_packet(packet)) % self.tcb) + return False + + if "A" == packet["TCP"].sprintf('%TCP.flags%'): + self.tcb = packet["TCP"].seq + self.logger.debug(("Synchronizing TCB (%d) on A packet " + actions.packet.Packet._str_packet(packet)) % self.tcb) + return False + + # If we're tracking this packet stream + if packet["TCP"].seq == self.tcb: + self.tcb += len(self.get_payload(packet)) + else: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by censor 4") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" + + + + + + + + + + + diff --git a/censors/censor5.py b/censors/censor5.py new file mode 100644 index 0000000..5e67dd7 --- /dev/null +++ b/censors/censor5.py @@ -0,0 +1,87 @@ +""" +Censor 5 + +Designed to be run by the evaluator. + +TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to +the server only. +""" + +import actions.packet +import logging +import netifaces +# Disable scapy ::1 warnings +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + +from scapy.all import IP, TCP, wrpcap + +from censors.censor import Censor + + +class Censor5(Censor): + """ + TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to + server AND client. + """ + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.enabled = True + self.tcb = 0 + self.drop_all_from = None + self.num = 0 + self.censor_interfaces = netifaces.interfaces() + if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces: + self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr'] + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.num += 1 + + # Only censor TCP packets for now + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if "TCP" not in packet: + return False + + if packet["TCP"].sprintf('%TCP.flags%') == "S": + self.tcb = packet["TCP"].seq + 1 + self.logger.debug("Synchronizing TCB on packet " + actions.packet.Packet._str_packet(packet)) + return False + + if packet["TCP"].seq == self.tcb: + self.tcb += len(self.get_payload(packet)) + + else: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Censor 3 Error caught.") + return False + + def censor(self, scapy_packet): + """ + Send 5 resets to the server. + """ + server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst) + server_tcp_rst = TCP( + dport=scapy_packet[TCP].dport, + sport=scapy_packet[TCP].sport, + ack=scapy_packet[TCP].ack, + seq=scapy_packet[TCP].seq, + flags="R" + ) + server_tcp_rst.show() + server_rst = server_ip_rst / server_tcp_rst + + for _ in range(0, 5): + self.mysend(server_rst) diff --git a/censors/censor6.py b/censors/censor6.py new file mode 100644 index 0000000..fe793eb --- /dev/null +++ b/censors/censor6.py @@ -0,0 +1,63 @@ +""" +Censor 6 is a IP dropping TCB Teardown censor. It tears down the TCB on any +FIN or RST packet. +Does not check if the ports are correct for the FIN/RST. + +""" +import actions.packet +import logging +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor6(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = 0 + self.drop_all_from = None + + def check_censor(self, packet, verbose=False): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Some stacks send RA to tear down a connection + if packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA", "F"]: + self.tcb = None + self.logger.debug(("Tearing down TCB on packet " + actions.packet.Packet._str_packet(packet))) + return False + + if self.tcb is None: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 6") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" diff --git a/censors/censor7.py b/censors/censor7.py new file mode 100644 index 0000000..f6a3d3d --- /dev/null +++ b/censors/censor7.py @@ -0,0 +1,74 @@ +""" +Censor 7 is a IP dropping TCB Teardown censor. It only tears down a TCB +if the full tuple of the TCB matches (src, dst, sport, dport). +Does not check if the SEQ/ACK are in window for the FIN/RST. +""" + +import logging +import actions.packet +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor7(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = {} + self.drop_all_from = None + + def check_censor(self, packet, verbose=False): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Clients can close connections with any of these flags + if packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA", "F"]: + # If a TCB has already been setup, check that this packet matches it + if self.tcb and \ + packet["IP"].src in self.tcb["ips"] and \ + packet["IP"].dst in self.tcb["ips"] and \ + packet["TCP"].dport in self.tcb["ports"] and \ + packet["TCP"].sport in self.tcb["ports"]: + + self.tcb = None + self.logger.debug(("Tearing down TCB on packet " + actions.packet.Packet._str_packet(packet))) + return False + + elif not self.tcb and self.tcb is not None: + self.tcb["ips"] = [packet["IP"].src, packet["IP"].dst] + self.tcb["ports"] = [packet["TCP"].sport, packet["TCP"].dport] + + if self.tcb is None: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 7") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" diff --git a/censors/censor8.py b/censors/censor8.py new file mode 100644 index 0000000..90a4d05 --- /dev/null +++ b/censors/censor8.py @@ -0,0 +1,78 @@ +""" +Censor 8 is a IP dropping TCB Teardown censor. It only tears down a TCB +if the full tuple of the TCB matches (src, dst, sport, dport, seq). +""" + +import logging +import actions.packet +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor8(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = {} + self.drop_all_from = None + + def check_censor(self, packet, verbose=False): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + if packet["TCP"].sprintf('%TCP.flags%') == "S" and not self.tcb and self.tcb is not None: + self.tcb["ips"] = [packet["IP"].src, packet["IP"].dst] + self.tcb["ports"] = [packet["TCP"].sport, packet["TCP"].dport] + self.tcb["seq"] = packet["TCP"].seq + 1 + self.logger.debug("Synchronizing TCB on packet " + actions.packet.Packet._str_packet(packet)) + return False + # TCB teardown + elif packet["TCP"].sprintf('%TCP.flags%') == "R" or packet["TCP"].sprintf('%TCP.flags%') == "F": + # If a TCB has already been setup, check that this packet matches it + if self.tcb and \ + packet["IP"].src in self.tcb["ips"] and \ + packet["IP"].dst in self.tcb["ips"] and \ + packet["TCP"].dport in self.tcb["ports"] and \ + packet["TCP"].sport in self.tcb["ports"] and \ + packet["TCP"].seq == self.tcb["seq"]: + + self.tcb = None + self.logger.debug(("Tearing down TCB on packet " + actions.packet.Packet._str_packet(packet))) + return False + + if self.tcb is None: + self.logger.debug("Ignoring packet: " + actions.packet.Packet._str_packet(packet)) + return False + elif "seq" in self.tcb and packet["TCP"].seq == self.tcb["seq"]: + self.tcb["seq"] += len(self.get_payload(packet)) + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 8") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" diff --git a/censors/censor8b.py b/censors/censor8b.py new file mode 100644 index 0000000..6ba1c43 --- /dev/null +++ b/censors/censor8b.py @@ -0,0 +1,159 @@ +""" +Censor 8b is a RST censor designed to mimic TCB teardown GFW behavior. It +tracks multiple connections using TCBs, but does not enter a TCB resynchronization +state if a RST or FIN, it simply tears down. It creates new TCBs for connections it is not +yet aware of, but does not check the checksums of incoming packets. +""" + +import actions.packet +import netifaces +from censors.censor import Censor +from scapy.all import IP, TCP + + +class Censor8b(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcbs = [] + self.flagged_ips = [] + self.censor_interfaces = netifaces.interfaces() + if(len(self.censor_interfaces) > 1) and 'eth0' in self.censor_interfaces: + self.censor_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr'] + + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. + Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: %s" % actions.packet.Packet._str_packet(packet)) + if packet["IP"].src in self.flagged_ips: + self.logger.debug("Content from a flagged IP detected %s..." % packet["IP"].src) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # Throw away packets that have an invalid dataofs + if packet["TCP"].dataofs < 5: + return False + + # If we are in a resynchronization state, or we do not yet have a connection and a new one + # is being created, add or update a TCB + tcb = self.get_matching_tcb(packet) + if (not tcb and packet["TCP"].sprintf('%TCP.flags%') in ["S"]): + # Check if we've been tracking a connection for this ip:port <-> ip:port already, + # so we can just replace that tcb with updated info + tcb = self.get_partial_tcb(packet) + if tcb is None: + self.logger.debug("Making a new TCB for packet %s" % actions.packet.Packet._str_packet(packet)) + tcb = {} + + tcb["src"] = packet["IP"].src + tcb["dst"] = packet["IP"].dst + tcb["sport"] = packet["TCP"].sport + tcb["dport"] = packet["TCP"].dport + tcb["seq"] = packet["TCP"].seq + # If we're synchronizing on a SYN flag, need to add 1. + if packet["TCP"].sprintf('%TCP.flags%') in ["S"]: + tcb["seq"] += 1 + else: + tcb["seq"] += len(self.get_payload(packet)) + + self.tcbs.append(tcb) + self.logger.debug("Synchronizing a TCB (%s) on packet %s " % (str(tcb), actions.packet.Packet._str_packet(packet))) + return False + # If connection is getting torn down + elif tcb and packet["TCP"].sprintf('%TCP.flags%') in ["R", "RA"]: + self.tcbs.remove(tcb) + self.logger.debug(("Deleting TCB for packet %s" % actions.packet.Packet._str_packet(packet))) + return False + + if not tcb: + self.logger.debug("No TCB matches packet.") + return False + + # Keep the TCB up to date + tcb["seq"] += len(self.get_payload(packet)) + + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: %s" % actions.packet.Packet._str_packet(packet)) + return True + + return False + except Exception: + self.logger.exception("Exception caught by Censor 8b") + return False + + def censor(self, scapy_packet): + """ + Adds client and server IPs to flagged IP list. + """ + if scapy_packet["IP"].src not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].src) + self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].src) + if scapy_packet["IP"].dst not in self.flagged_ips: + self.flagged_ips.append(scapy_packet["IP"].dst) + self.logger.debug("Marking IP %s for dropping..." % scapy_packet["IP"].dst) + + client_ip_rst = IP(src=scapy_packet[IP].dst, dst=scapy_packet[IP].src) + client_tcp_rst = TCP( + dport=scapy_packet[TCP].sport, + sport=scapy_packet[TCP].dport, + ack=scapy_packet[TCP].seq+len(str(scapy_packet[TCP].payload)), + seq=scapy_packet[TCP].ack, + flags="R" + ) + client_rst = client_ip_rst / client_tcp_rst + + server_ip_rst = IP(src=self.censor_ip, dst=scapy_packet[IP].dst) + server_tcp_rst = TCP( + dport=scapy_packet[TCP].dport, + sport=scapy_packet[TCP].sport, + ack=scapy_packet[TCP].ack, + seq=scapy_packet[TCP].seq, + flags="R" + ) + server_tcp_rst.show() + server_rst = server_ip_rst / server_tcp_rst + + for _ in range(0, 5): + self.mysend(client_rst) + self.mysend(server_rst) + + return "accept" + + def get_matching_tcb(self, packet): + """ + Checks if the packet matches the stored TCB. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"] and \ + packet["TCP"].seq == tcb["seq"]): + return tcb + return None + + def get_partial_tcb(self, packet): + """ + Checks if the packet matches an existing connection, regardless if the SEQ/ACK + are correct. + """ + for tcb in self.tcbs: + self.logger.debug("Checking %s against packet %s for partial match" % (str(tcb), actions.packet.Packet._str_packet(packet))) + + if (packet["IP"].src == tcb["src"] and \ + packet["IP"].dst == tcb["dst"] and \ + packet["TCP"].sport == tcb["sport"] and \ + packet["TCP"].dport == tcb["dport"]): + return tcb + return None diff --git a/censors/censor9.py b/censors/censor9.py new file mode 100644 index 0000000..736ea17 --- /dev/null +++ b/censors/censor9.py @@ -0,0 +1,106 @@ +""" +Censor 9 is a IP dropping TCB Teardown censor. It does not tear down its TCB, +but it will resynchronize it's TCB if a RST or FIN is sent if the full tuple +of the TCB matches (src, dst, sport, dport, seq). + +More closely mimics GFW behavior. +""" + +import logging +import actions.packet +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +from scapy.all import IP, TCP + +from censors.censor import Censor + + +class Censor9(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + self.forbidden = forbidden + self.tcb = {} + self.drop_all_from = None + self.resynchronize = False + + def check_censor(self, packet, verbose=False): + """ + Check if the censor should run against this packet. Returns true or false. + """ + try: + self.logger.debug("Inbound packet to censor: " + actions.packet.Packet._str_packet(packet)) + if self.drop_all_from == packet["IP"].src: + self.logger.debug("Dropping all from this IP %s..." % self.drop_all_from) + return True + + # Only censor TCP packets for now + if "TCP" not in packet: + return False + + # If we are in a resynchronization state, or we do not yet have a connection and a new one + # is being created, definte the TCB + if self.resynchronize or (not self.tcb and packet["TCP"].sprintf('%TCP.flags%') == "S"): + self.tcb["src"] = packet["IP"].src + self.tcb["dst"] = packet["IP"].dst + self.tcb["sport"] = packet["TCP"].sport + self.tcb["dport"] = packet["TCP"].dport + self.tcb["seq"] = packet["TCP"].seq + # If we're synchronizing on a SYN flag, need to add 1. + if packet["TCP"].sprintf('%TCP.flags%') == "S": + self.tcb["seq"] += 1 + else: + self.tcb["seq"] += len(self.get_payload(packet)) + + self.resynchronize = False + self.logger.debug("Synchronizing TCB on packet " + actions.packet.Packet._str_packet(packet)) + return self.check_forbidden(packet) + + # If connection is getting torn down + elif self.tcb_matches(packet) and \ + (packet["TCP"].sprintf('%TCP.flags%') == "R" or \ + packet["TCP"].sprintf('%TCP.flags%') == "F"): + self.resynchronize = True + self.logger.debug(("Entering resynchronization state on packet " + actions.packet.Packet._str_packet(packet))) + + if not self.tcb_matches(packet): + self.logger.debug("TCB does not match packet.") + return False + # Keep the TCB up to date + elif "seq" in self.tcb: + self.tcb["seq"] += len(self.get_payload(packet)) + + return self.check_forbidden(packet) + + except Exception: + self.logger.exception("Exception caught by Censor 9") + return False + + def censor(self, scapy_packet): + """ + Marks this IP to be dropped in the future and drops this packet. + """ + self.drop_all_from = scapy_packet["IP"].src + self.logger.debug("Marking IP %s for dropping..." % self.drop_all_from) + return "drop" + + def check_forbidden(self, packet): + """ + Checks if a packet contains forbidden words. + """ + # Check if any forbidden words appear in the packet payload + for keyword in self.forbidden: + if keyword in self.get_payload(packet): + self.logger.debug("Packet triggered censor: " + actions.packet.Packet._str_packet(packet)) + return True + return False + + def tcb_matches(self, packet): + """ + Checks if the packet matches the stored TCB. + """ + self.logger.debug(self.tcb) + return not self.tcb or (self.tcb and \ + packet["IP"].src == self.tcb["src"] and \ + packet["IP"].dst == self.tcb["dst"] and \ + packet["TCP"].sport == self.tcb["sport"] and \ + packet["TCP"].dport == self.tcb["dport"] and \ + packet["TCP"].seq == self.tcb["seq"]) diff --git a/censors/censor_driver.py b/censors/censor_driver.py new file mode 100644 index 0000000..f4bbd4d --- /dev/null +++ b/censors/censor_driver.py @@ -0,0 +1,94 @@ +import argparse +import importlib +import inspect +import os +import traceback +import sys + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(BASEPATH) + +if PROJECT_ROOT not in sys.path: + sys.path.append(PROJECT_ROOT) + +import censors.censor + +CENSORS = {} + +def get_censors(): + """ + Dynamically imports all of the Censors classes in this directory. + """ + global CENSORS + if CENSORS: + return CENSORS + + collected_censors = {} + for censor_file in os.listdir(os.path.dirname(os.path.abspath(__file__))): + if not censor_file.endswith(".py"): + continue + censor_file = censor_file.replace(".py", "") + importlib.import_module("censors."+censor_file) + def check_censor(o): + return inspect.isclass(o) and issubclass(o, censors.censor.Censor) and o != censors.censor.Censor + clsmembers = inspect.getmembers(sys.modules["censors."+censor_file], predicate=check_censor) + if clsmembers: + name, censor_class = clsmembers[0] + if censor_class(0, [], None, None, None, None).enabled: + collected_censors[name.lower()] = censor_class + + CENSORS = collected_censors + return collected_censors + + +def get_args(): + """ + Sets up argparse and collects arguments. + """ + parser = argparse.ArgumentParser(description='The server, run by the evaluator.') + parser.add_argument('--port', type=int, action='store', help="Server port", + required=True) + parser.add_argument('--queue', type=int, action='store', help="NFQueue number to use", + required=True) + parser.add_argument('--environment-id', action='store', help="ID of the current environment", + required=True) + parser.add_argument('--censor', action='store', help="censor to deploy", required=True) + parser.add_argument('--forbidden', action='store', default='ultrasurf', help="word to censor") + parser.add_argument('--output-directory', action='store', help="Where to write logs", + required=True) + parser.add_argument('--log', action='store', default="debug", + choices=("debug", "info", "warning", "critical", "error"), + help="Sets the log level") + + return parser.parse_args() + + +def main(args): + """ + Starts the given censor. + """ + try: + censors = get_censors() + censor_name = args["censor"].lower() + + if censor_name not in censors: + print("ERROR: Unknown censor.") + return None + censor_cls = censors[censor_name] + censor = censor_cls(args["environment_id"], + [args["forbidden"].encode('utf-8')], + args["output_directory"], + args["log"], + args["port"], + args["queue"]) + print("Censor %s starting." % censor_name) + censor.start() + except Exception as e: + print(e) + traceback.print_exc() + + +# Note that this code can be removed - this is how the +# evaluator runs the censor for tests in the Docker environment +if __name__ == "__main__": + main(vars(get_args())) diff --git a/censors/censors.txt b/censors/censors.txt new file mode 100644 index 0000000..869dccf --- /dev/null +++ b/censors/censors.txt @@ -0,0 +1,22 @@ +1. TCP Censor that synchronizes on first SYN only, works 100% of the time, drops all packets after TCP forbidden content + - [flag:S]dup-tamper{TCP:syn:corrupt} +2. TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to client after TCP forbidden content + - [flag:R]drop OR [flag:S]dup-tamper{TCP:syn:corrupt} +3. TCP Censor that synchronizes on first SYN only, works 100% of the time, sends 5 RSTs to server AND client after TCP forbidden content + - [flag:S]dup-tamper{TCP:syn:corrupt} +4. TCP Censor that synchronizes on first SYN, ACK only, works 100% of the time, drops all packets after TCP forbidden content + - [flag:S,A]dup-tamper{TCP:syn:corrupt} +5. TCP Censor that synchronizes on first SYN and every 5 packets after that + - [flag:S]dup-tamper{TCP:syn:corrupt},[cyclic:4]dup-tamper{TCP:syn:corrupt} +6. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached only to client/server IP is not torn down +7. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached to client/server IP and port is not torn down +8. TCP Censor that works 100% of the time, drops all packets after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ is not torn down + - [field:TCP:flags:A]-duplicate-tamper{TCP:flags:replace:R}-tamper{TCP:chksum:replace:4444}-| \/ +9. TCP Censor that works 100% of the time, tracks only 1 connection, drops all packets after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ if TCB matches + - Need to force re-sync w/ FIN/RST and then send insertion packet +10. TCP Censor that can track multiple connections, can resynchronize its TCBs, checks packet checksums, works 100% of the time, drops all packets from either IP address after TCP forbidden content if a TCB attached to client/server IP, port, and in-window SEQ if TCB matches + - Need to force re-sync w/ FIN/RST and then send insertion packet with a correct chksum + +UDP: +1. Censor that can inject responses to DNS requests + - [default value in packet]drop diff --git a/censors/dummy.py b/censors/dummy.py new file mode 100644 index 0000000..340ad25 --- /dev/null +++ b/censors/dummy.py @@ -0,0 +1,26 @@ +""" +Dummy + +Designed to be run by the evaluator. + +Censors nothing - dummy censor for infrastructure testing. +""" + +from censors.censor import Censor + + +class Dummy(Censor): + def __init__(self, environment_id, forbidden, log_dir, log_level, port, queue_num): + Censor.__init__(self, environment_id, log_dir, log_level, port, queue_num) + + def check_censor(self, packet): + """ + Check if the censor should run against this packet. Returns False for dummy censor. + """ + return False + + def censor(self, scapy_packet): + """ + Does nothing. + """ + return False diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..e9ea6a8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.6-stretch +# Set the root password in the container so we can use it +RUN echo "root:Docker!" | chpasswd +RUN apt-get -y update +RUN apt-get -y install libnetfilter-queue-dev iptables tcpdump netcat net-tools git graphviz openssh-server +# Enable root SSH login for client testing +RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config + +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install tshark +ENV PATH="/usr/sbin:${PATH}" +RUN pip install netfilterqueue requests dnspython anytree graphviz netifaces paramiko tld docker scapy==2.4.3 psutil +ENTRYPOINT ["/bin/bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..821380c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,10 @@ +# Geneva Docker + +This implements the Docker base image for Geneva. You can run the base image with the below python: +``` +import os +import docker + +docker_client = docker.from_env() +docker_client.containers.run('base', detach=True, privileged=True, volumes={os.path.abspath(os.getcwd()): {"bind" : "/code", "mode" : "rw"}}, tty=True, remove=True, name="test") +``` diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api/actions/action.rst b/docs/api/actions/action.rst new file mode 100644 index 0000000..93cc994 --- /dev/null +++ b/docs/api/actions/action.rst @@ -0,0 +1,7 @@ +geneva.actions.action +===================== + +.. automodule:: action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/drop.rst b/docs/api/actions/drop.rst new file mode 100644 index 0000000..731dafb --- /dev/null +++ b/docs/api/actions/drop.rst @@ -0,0 +1,7 @@ +geneva.actions.drop +=================== + +.. automodule:: drop + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/duplicate.rst b/docs/api/actions/duplicate.rst new file mode 100644 index 0000000..ae3c1dd --- /dev/null +++ b/docs/api/actions/duplicate.rst @@ -0,0 +1,7 @@ +geneva.actions.duplicate +======================== + +.. automodule:: duplicate + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/fragment.rst b/docs/api/actions/fragment.rst new file mode 100644 index 0000000..6e3fbb8 --- /dev/null +++ b/docs/api/actions/fragment.rst @@ -0,0 +1,7 @@ +geneva.actions.fragment +======================= + +.. automodule:: fragment + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/layer.rst b/docs/api/actions/layer.rst new file mode 100644 index 0000000..f499b0c --- /dev/null +++ b/docs/api/actions/layer.rst @@ -0,0 +1,7 @@ +geneva.actions.layer +==================== + +.. automodule:: layer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/packet.rst b/docs/api/actions/packet.rst new file mode 100644 index 0000000..d2c96f3 --- /dev/null +++ b/docs/api/actions/packet.rst @@ -0,0 +1,7 @@ +geneva.actions.packet +===================== + +.. automodule:: packet + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/sleep.rst b/docs/api/actions/sleep.rst new file mode 100644 index 0000000..7ce2acd --- /dev/null +++ b/docs/api/actions/sleep.rst @@ -0,0 +1,7 @@ +geneva.actions.sleep +==================== + +.. automodule:: sleep + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/strategy.rst b/docs/api/actions/strategy.rst new file mode 100644 index 0000000..9b32cdb --- /dev/null +++ b/docs/api/actions/strategy.rst @@ -0,0 +1,7 @@ +geneva.actions.strategy +======================= + +.. automodule:: strategy + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/tamper.rst b/docs/api/actions/tamper.rst new file mode 100644 index 0000000..bd5a01b --- /dev/null +++ b/docs/api/actions/tamper.rst @@ -0,0 +1,7 @@ +geneva.actions.tamper +===================== + +.. automodule:: tamper + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/trace.rst b/docs/api/actions/trace.rst new file mode 100644 index 0000000..1822b04 --- /dev/null +++ b/docs/api/actions/trace.rst @@ -0,0 +1,7 @@ +geneva.actions.trace +==================== + +.. automodule:: trace + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/tree.rst b/docs/api/actions/tree.rst new file mode 100644 index 0000000..7872d6a --- /dev/null +++ b/docs/api/actions/tree.rst @@ -0,0 +1,7 @@ +geneva.actions.tree +==================== + +.. automodule:: tree + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/trigger.rst b/docs/api/actions/trigger.rst new file mode 100644 index 0000000..918c597 --- /dev/null +++ b/docs/api/actions/trigger.rst @@ -0,0 +1,7 @@ +geneva.actions.trigger +====================== + +.. automodule:: trigger + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/actions/utils.rst b/docs/api/actions/utils.rst new file mode 100644 index 0000000..ad512fb --- /dev/null +++ b/docs/api/actions/utils.rst @@ -0,0 +1,7 @@ +geneva.actions.utils +==================== + +.. automodule:: utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/engine.rst b/docs/api/engine.rst new file mode 100644 index 0000000..2c0f30e --- /dev/null +++ b/docs/api/engine.rst @@ -0,0 +1,7 @@ +geneva.engine +============= + +.. automodule:: engine + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/evaluator.rst b/docs/api/evaluator.rst new file mode 100644 index 0000000..87f9edd --- /dev/null +++ b/docs/api/evaluator.rst @@ -0,0 +1,7 @@ +geneva.evaluator +================= + +.. automodule:: evaluator + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/evolve.rst b/docs/api/evolve.rst new file mode 100644 index 0000000..39cc600 --- /dev/null +++ b/docs/api/evolve.rst @@ -0,0 +1,7 @@ +geneva.evolve +============= + +.. automodule:: evolve + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/dns.rst b/docs/api/plugins/dns.rst new file mode 100644 index 0000000..54eea9d --- /dev/null +++ b/docs/api/plugins/dns.rst @@ -0,0 +1,17 @@ +geneva.plugins.dns +============================ + +.. automodule:: plugins.dns.client + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: plugins.dns.server + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: plugins.dns.plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/echo.rst b/docs/api/plugins/echo.rst new file mode 100644 index 0000000..9aa17f3 --- /dev/null +++ b/docs/api/plugins/echo.rst @@ -0,0 +1,12 @@ +geneva.plugins.echo +============================ + +.. automodule:: plugins.echo.client + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: plugins.echo.server + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/http.rst b/docs/api/plugins/http.rst new file mode 100644 index 0000000..eaf76c4 --- /dev/null +++ b/docs/api/plugins/http.rst @@ -0,0 +1,17 @@ +geneva.plugins.http +============================ + +.. automodule:: plugins.http.client + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: plugins.http.server + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: plugins.http.plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/plugin.rst b/docs/api/plugins/plugin.rst new file mode 100644 index 0000000..247a78c --- /dev/null +++ b/docs/api/plugins/plugin.rst @@ -0,0 +1,7 @@ +geneva.plugins.plugin +===================== + +.. automodule:: plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/plugin_client.rst b/docs/api/plugins/plugin_client.rst new file mode 100644 index 0000000..bbe3178 --- /dev/null +++ b/docs/api/plugins/plugin_client.rst @@ -0,0 +1,7 @@ +geneva.plugins.plugin_client +============================ + +.. automodule:: plugin_client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/plugin_server.rst b/docs/api/plugins/plugin_server.rst new file mode 100644 index 0000000..9e88c36 --- /dev/null +++ b/docs/api/plugins/plugin_server.rst @@ -0,0 +1,7 @@ +geneva.plugins.plugin_server +============================ + +.. automodule:: plugin_server + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/plugins/sni.rst b/docs/api/plugins/sni.rst new file mode 100644 index 0000000..89b1ba3 --- /dev/null +++ b/docs/api/plugins/sni.rst @@ -0,0 +1,7 @@ +geneva.plugins.sni +============================ + +.. automodule:: plugins.sni.client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..b4bf827 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,78 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(1, os.path.dirname(os.path.abspath('.'))) +sys.path.insert(2, os.path.join(os.path.dirname(os.path.abspath('.')), "actions")) +sys.path.insert(3, os.path.join(os.path.dirname(os.path.abspath('.')), "plugins")) +# Hack so the HTTP plugin will import +sys.path.insert(4, os.path.join(os.path.dirname(os.path.abspath('.')), "plugins", "http")) +import sphinx_rtd_theme + + +# -- Project information ----------------------------------------------------- + +project = 'geneva' +copyright = '2020, Kevin Bock' +author = 'Kevin Bock' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx_rtd_theme", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.autosectionlabel" +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +autodoc_member_order = 'groupwise' + +# Don't skip __init__ +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + +def setup(app): + app.connect("autodoc-skip-member", skip) + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +pygments_style = 'colorful' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'] } + diff --git a/docs/extending/actions.rst b/docs/extending/actions.rst new file mode 100644 index 0000000..3248304 --- /dev/null +++ b/docs/extending/actions.rst @@ -0,0 +1,127 @@ +Defining New Actions +===================== + +It is simple to add a new packet-level action to Geneva. + +Let us assume we are adding a new action, called "mytamper", which simply sets the :code:`ipid` field of a packet. +Our action will take 1 packet and return 1 packet, and we'll start by making it always set the IPID to 1. + +We will subclasss the :code:`Action` class, and specify an :code:`__init__` method and a :code:`run` method. + +In the :code:`__init__` method, we will specify that our action's name is 'mytamper' and can run in :code:`both` inbound and outbound. +Then, in the :code:`run()` method, we will use Geneva's packet API to set the :code:`ipid` field to 1 and simply return the packet. + +.. code-block:: python + + from actions.action import Action + + class MyTamperAction(Action): + """ + Geneva action to set the IPID to 1. + """ + # Controls frequency with which this action is chosen by the genetic algorithm + # during mutation + frequency = 0 + + def __init__(self, environment_id=None): + Action.__init__(self, "mytamper", "both") + + def run(self, packet, logger): + """ + The mytamper action returns a modified packet as the left child. + """ + logger.debug(" - Changing IPID field to 1") + packet.set("IP", "ipid", 1) + return packet, None + +And that's it! Now, we can specify this action in our normal strategy DNA: Geneva will discover it dynamically on startup, import it, and we can use it. + +Adding Parameters +^^^^^^^^^^^^^^^^^ + +Let's now assume we want to make our action take parameters. We will add two new methods: :code:`parse()` and :code:`__str__()`. +We'll start by adding a new instance variable :code:`self.ipid_value`. + +.. code-block:: python + + def __init__(self, environment_id=None, ipid_value=1): + Action.__init__(self, "mytamper", "both") + self.ipid_value = ipid_value + +Next, we'll add the :code:`__str__` method so when our action is printed in the strategy DNA, its components are too: + +.. code-block:: python + + def __str__(self): + """ + Returns a string representation. + """ + s = Action.__str__(self) + s += "{%g}" % self.ipid_value + return s + +Finally, we'll add the :code:`parse()` method so we can parse the value from a string strategy DNA to a live action. + +.. code-block:: python + + def parse(self, string, logger): + """ + Parses a string representation for this object. + """ + try: + if string: + self.ipid_value = float(string) + except ValueError: + logger.exception("Cannot parse ipid_value %s" % string) + return False + + return True + +Putting it all together: + +.. code-block:: python + + from actions.action import Action + + class MyTamperAction(Action): + """ + Geneva action to set the IPID to 1. + """ + # Controls frequency with which this action is chosen by the genetic algorithm + # during mutation + frequency = 0 + + def __init__(self, environment_id=None, ipid_value=1): + Action.__init__(self, "mytamper", "both") + self.ipid_value = ipid_value + + def run(self, packet, logger): + """ + The mytamper action returns a modified packet as the left child. + """ + logger.debug(" - Changing IPID field to 1") + packet.set("IP", "ipid", 1) + return packet, None + + def __str__(self): + """ + Returns a string representation. + """ + s = Action.__str__(self) + s += "{%g}" % self.ipid_value + return s + + def parse(self, string, logger): + """ + Parses a string representation for this object. + """ + try: + if string: + self.ipid_value = float(string) + except ValueError: + logger.exception("Cannot parse ipid_value %s" % string) + return False + + return True + +And we're done! Now, we can write strategies like: :code:`[TCP:flags:PA]-mytamper{10}-|`, and any TCP packet with the flags field set to :code:`PA` will have its :code:`ipid` field set to 10. diff --git a/docs/extending/contributing.rst b/docs/extending/contributing.rst new file mode 100644 index 0000000..fff80ac --- /dev/null +++ b/docs/extending/contributing.rst @@ -0,0 +1,6 @@ +Contributing +============= + +Contributions are welcome! You are encouraged to fork the repository, open Github issues with us, +or just make a pull request. If you are interested in becoming more involved with the team or +development, checkout our website a `https://censorship.ai `_ and drop us a line! diff --git a/docs/extending/layers.rst b/docs/extending/layers.rst new file mode 100644 index 0000000..06469e3 --- /dev/null +++ b/docs/extending/layers.rst @@ -0,0 +1,2 @@ +Exposing New Protocols +====================== diff --git a/docs/extending/plugins.rst b/docs/extending/plugins.rst new file mode 100644 index 0000000..4557403 --- /dev/null +++ b/docs/extending/plugins.rst @@ -0,0 +1,276 @@ +Adding New Plugins +================== + +This section will describe the process to add a new application plugin to Geneva. +Application plugins serve as the fitness function for Geneva during evolution, +and allow it to evolve strategies to defeat certain types of censorship. + +Plugins are run by the Evaluator; if you have not yet read how the Evaluator +works, see the :ref:`Strategy Evaluation` section. + +There are three types of plugins: clients, servers, and overriding plugins. +A developer can choose to implement any one of, or all three of these plugins. + +For this section, we will build an example plugin and walk through the existing +plugins to tour through the plugin API. + +Plugins are expected to be in the :code:`plugins/` folder in geneva's repo. The +folder name is the plugin name, and Geneva will discover these automatically. +Plugins are specified to the evaluator or evolve with the :code:`--test-type` +flag. Within the plugin folder, plugins must adhere to the following naming +scheme: + +- :code:`client.py` - for plugin clients +- :code:`server.py` - for plugin servers [optional] +- :code:`plugin.py` - for an overriding plugin to customize logic [optional] + +:code:`server.py` and :code:`plugin.py` are optional. The server plugin is +required to do server-side evaluation, but the overriding plugin definition is +only required to if a developer wishes to override the evaluator's default +behavior. + +If an overriding plugin is provided, the evaluator will simply invoke it at the +start of strategy evaluation and the overriding plugin will be responsible for +calling the client and server. This section will assume that no overriding +plugin is specified to describe the evaluator's default behavior with plugins, +and cover use cases for overriding plugins at the end. + +Depending on the evaluation setup, some (or all) of these plugins will be used +during evaluation. For example, during an exclusively client-side evaluation, +only the client plugin is needed. + +Client Plugins +^^^^^^^^^^^^^^ + +During exclusively client-side evolution, the evaluator will start the engine +with the strategy under evaluation, and then run the client plugin. During +server-side evolution, the evaluator will run the engine on the server-side, +start the server plugin, and then start the client plugin via an SSH session to +the remote client worker. (See :ref:`Adding a Worker` on how external workers +can be used). + +The client plugin subclasses from the PluginClient object. To tour through the +API, we will walk through the development of a custom client plugin. + +Writing Our Own +~~~~~~~~~~~~~~~ + +Let us write a fitness function to test Iran's whitelisting system. + +Iran's protocol whitelister was a recently deployed new censorship mechanism to +censor non-whitelisted protocols on certain ports (53, 80, 443). We deployed +Geneva against the whitelister, and discovered multiple ways to evade it in just +one evolution of the genetic algorithm. (The results of that investigation is +located `here `_). + +The whitelister worked by checking the first 2 packets of a flow, and if they +did not match a fingerprint, it would destroy the flow. + +In order to run Geneva against whitelister, we will define a client plugin that +will try to trigger the whitelister and record whether the whitelister +successfully censored its connection or not. + +First, let's make a new folder in the :code:`plugins/` directory called +"whitelister". We'll create a "client.py" and create the plugin object as +a subclass of ClientPlugin. + +.. code-block:: python + + class WhitelisterClient(ClientPlugin): + """ + Defines the whitelister client. + """ + name = "whitelister" + + def __init__(self, args): + """ + Initializes the whitelister client. + """ + ClientPlugin.__init__(self) + self.args = args + + +Done! Next, let's define argument parsing for this plugin. Geneva uses a +pass-through system of argument parsing: when command-line arguments are +specified, evolve.py parses the options it knows and passes the rest to the +evaluator. The evaluator prases the options it knows, and passes the list to the +plugins. This allows developers to easily add their own arguments just to their +plugin and use them from the command-line without changing any of the +intermediate code. + +In this case, we need our client to make a TCP connection to a server located +outside of Iran to send our whitelister triggering messages to. Let's add an +argument so the user can specify which server to connect to. + +We can do this by adding a :code:`get_args` static method. The evaluator will +call this method when the plugin is created and give it the full command line +list, so the plugin is free to parse it how it chooses. For this example, we +will use the standard :code:`argparse` library. + +Since the superclass also defines args, we'll pass the command line list up to +the super class as well to collect those arguments. + +.. code-block:: python + + @staticmethod + def get_args(command): + """ + Defines args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='Whitelister Client') + + parser.add_argument('--server', action='store', help="server to connect to") + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + +Now, we just need to define a :code:`run()` method. The :code:`run()` method is +called by the evaluator to run the plugin. It provides the parsed arguments, a +logger to log with, and a reference to an instance of the strategy engine that +is running the strategy (see :ref:`Engine` for more information on how the +engine works.) + +Let's start by defining the run method. We'll pull out the argument for the +server we defined earlier, connect to it with a python socket, and then just +send "G", "E", and "T" in separate messages to trigger the whitelister. Since +the whitelister censors connections by blackholing them, if the strategy failed +to defeat the whitelister, we would expect our network connection to timeout; +if we can send our messages and get a response from the server, the strategy +under evaluation may have defeated the whitelister. + +.. code-block:: python + + def run(self, args, logger, engine=None): + """ + Try to open a socket, send two messages, and see if the messages + time out. + """ + fitness = 0 + port = int(args["port"]) + server = args["server"] + try: + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.settimeout(3) + client.connect((server, port)) + client.sendall(b"G") + time.sleep(0.25) + client.sendall(b"E") + time.sleep(0.25) + client.sendall(b"T\r\n\r\n") + server_data = client.recv(1024) + logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore')) + if server_data: + fitness += 100 + else: + fitness -= 90 + client.close() + # ... + +Now we just need to define the error handling for this code. This is critical to +the fitness function: we want to kill off strategies that damage the underlying +TCP connection, so Geneva does not waste time searching this space of +strategies. + +Our goal is to set the fitness metric such that a *censorship event has a higher +fitness than the strategy damaging the connection*. Since we can distinguish +these cases based on the socket error, we will set a lower fitness if any other +exception is raised besides the timeout. + +Lastly, we'll inflate the numerical fitness metric to make it a larger number. +The evaluator does additional punishments to the fitness score based on the +strategy (see :ref:`Strategy Evaluation`), so we want the number to be +sufficiently large to not push succeeding strategies to negative numbers. + +.. code-block:: python + + except socket.timeout: + logger.debug("Client: Timeout") + fitness -= 90 + except socket.error as exc: + fitness -= 100 + logger.exception("Socket error caught in client echo test.") + finally: + logger.debug("Client finished whitelister test.") + return fitness * 4 + + +Putting it all together: + +.. code-block:: python + + class WhitelisterClient(ClientPlugin): + """ + Defines the whitelister client. + """ + name = "whitelister" + + def __init__(self, args): + """ + Initializes the whitelister client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='Whitelister Client') + + parser.add_argument('--server', action='store', help="server to connect to") + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to open a socket, send two messages, and see if the messages + time out. + """ + fitness = 0 + port = int(args["port"]) + server = args["server"] + try: + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.settimeout(3) + client.connect((server, port)) + client.sendall(b"G") + time.sleep(0.25) + client.sendall(b"E") + time.sleep(0.25) + client.sendall(b"T\r\n\r\n") + server_data = client.recv(1024) + logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore')) + if server_data: + fitness += 100 + else: + fitness -= 90 + client.close() + except socket.timeout: + logger.debug("Client: Timeout") + fitness -= 90 + except socket.error as exc: + fitness -= 100 + logger.exception("Socket error caught in client echo test.") + finally: + logger.debug("Client finished whitelister test.") + return fitness * 4 + +Server Plugins +^^^^^^^^^^^^^^ + +Coming soon! + +Override Plugins +^^^^^^^^^^^^^^^^ + +Coming soon! diff --git a/docs/howitworks/addingaworker.rst b/docs/howitworks/addingaworker.rst new file mode 100644 index 0000000..d3cf6cd --- /dev/null +++ b/docs/howitworks/addingaworker.rst @@ -0,0 +1,42 @@ +Adding a Worker +=============== + +Below is how to add a new external client worker to Geneva. An external client worker is an external, SSH-accessible machine under the control of the user running Geneva for the purpose of performing strategy evaluation from outside the censored regime. + +Geneva expects each of its worker to be defined in the :code:`workers` folder. + +For this section, let us assume we are trying to allow Geneva to use a new external worker located in China. + +First, make a new subfolder for that worker under the :code:`workers/` directory. + +.. code-block:: none + + # mkdir workers/test + # ls workers/ + example test + +Each worker is defined by a :code:`worker.json` file located inside its subfolder. + +The structure of the worker looks like this: + +.. code-block:: json + + { + "name": "test", + "ip": "", + "hostname": "", + "username": "user", + "password": null, + "port": 22, + "python": "python3", + "city": "Bejing", + "keyfile": "example.pem", + "country": "China", + "geneva_path": "~/geneva" + } + +If passwordless SSH is used, you can optionally specify a keyfile for it to SSH with. + +Once this is defined, we can specify :code:`--external-client test` during strategy evaluation, and the evaluator will SSH to this worker for training! + +.. note:: Remember, external client workers must have Geneva cloned to the directory specified in :code:`geneva_path` and depencies set up before use. diff --git a/docs/howitworks/engine.rst b/docs/howitworks/engine.rst new file mode 100644 index 0000000..8edb43b --- /dev/null +++ b/docs/howitworks/engine.rst @@ -0,0 +1,37 @@ +Engine +====== + +The strategy engine (:code:`engine.py`) applies a strategy to a network connection. The engine works by capturing all traffic to/from a specified port. Packets that match an active trigger are run through the associated action-tree, and packets that emerge from the tree are sent on the wire. + +.. code-block:: bash + + # python3 engine.py --server-port 80 --strategy "\/" --log debug + 2019-10-14 16:34:45 DEBUG:[ENGINE] Engine created with strategy \/ (ID bm3kdw3r) to port 80 + 2019-10-14 16:34:45 DEBUG:[ENGINE] Configuring iptables rules + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p tcp --sport 80 -j NFQUEUE --queue-num 1 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p tcp --dport 80 -j NFQUEUE --queue-num 2 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p udp --sport 80 -j NFQUEUE --queue-num 1 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p udp --dport 80 -j NFQUEUE --queue-num 2 + + +The engine also has a Python API for using it in your application. It can be used as a context manager or invoked in the background as a thread. +For example, consider the following simple application. + +.. code-block:: python + + import os + import engine + + # Port to run the engine on + port = 80 + # Strategy to use + strategy = "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/" + + # Create the engine in debug mode + with engine.Engine(port, strategy, log_level="debug") as eng: + os.system("curl http://example.com?q=ultrasurf") + +This script creates an instance of the engine with a specified strategy, and that strategy will be running for everything within the context manager. When the context manager exits, the engine will clean itself up. See the :code:`examples/` folder for more use cases of the engine. + +.. note:: Due to limitations of scapy and NFQueue, the engine cannot be used to communicate with localhost. + diff --git a/docs/howitworks/evaluation.rst b/docs/howitworks/evaluation.rst new file mode 100644 index 0000000..809ad91 --- /dev/null +++ b/docs/howitworks/evaluation.rst @@ -0,0 +1,70 @@ +Strategy Evaluation +=================== + +Strategy evaluation tries to answer the question, *"which censorship evasion +strategies should survive and propagate in the next generation?"* This is the +job of the evaluator (:code:`evaluator.py`). The :code:`evaluator` assigns a numerical +fitness score to each strategy based on a fitness function. The actual numerical +fitness score itself is unimportant: as long as a 'better' strategy has a higher +fitness score than a 'worse' strategy, our fitness function will be sound. +Specific fitness numbers will be used in this section as examples, but all that +matters is the _comparison_ between fitness scores. + +Since the goal of Geneva is to find a strategy that evades censorship, to test a +strategy, we can simply send a forbidden query through a censor with a strategy +running and see if the request gets censored. Geneva operates at the network +(TCP/IP) layer, so it is completely application agnostic: whether :code:`curl` or +Google Chrome is generating network traffic, the engine can capture and modify +it. + +Environment IDs +^^^^^^^^^^^^^^^ + +During each evaluation, every strategy under test is given a random identifier, called an 'environment id'. This is used to track each strategy during evaluation. As each strategy is evaluated, a log file is written out named after the environment ID. See :ref:`Logging` for more information on how logs are stored. + + +Plugins +^^^^^^^^ + +To allow for evolution for many different applications, Geneva has a system of +plugins (in `plugins/`). A plugin is used to drive an application to make a +forbidden request, and defines the fitness function for that application. These +plugins provide evaluator with a common interface to launch them and get the +fitness score back. + +Geneva provides some plugins with fitness functions for common protocols out of +the box. Whenever the evaluator or :code:`evolve.py` is run, you must specify +which plugin it should use with :code:`--test-type `, such as +:code:`--test-type http`. + +Plugins can define a client and optionally a server. The client will attempt to +make a forbidden connection through the censor to an external server, or +optionally if run from another computer, to an instance of the server plugin. + +To evaluate a strategy, the evaluator will initialize the engine with the strategy, +launch the plugin client, and record the fitness returned by the plugin. + +See :ref:`Adding New Plugins` for more details on how plugins work, how they can override behavior in the evaluator, and how to add new ones to Geneva. + +Fitness Functions +================= + +Unlike many other machine learning scenarios, we have no gradient to learn against; censors give us only a binary signal (censored or not) to learn from. Therefore, we can't just write a fitness function that will directly guide us to the answer. Instead, we will use the fitness function to encourage the genetic algorithm to search the space of strategies that keeps the TCP connection alive. + +As mentioned above, this guide might have specific fitness #s in examples, but the actual numeric values are not important - what is important is that a "bad" strategy gets a lower fitness number than a "good" strategy. We will use the fitness function to define a hierarchy of strategies. + +The comparison order used by Geneva, ordered from best to worst: + - Strategy that does not get censored and generates a minimal number of packets, no unused triggers, and minimal size + - Strategy that does not get censored, but has unused actions, is too large, or imparts overhead + - Strategy that gets censored + - Strategy that does not trigger on any packets but gets censored + - Strategy that breaks the underlying TCP connection + - Empty strategy + +Accomplishing this hierarchy is relatively straightforward - when evaluating a strategy, we assign a numerical fitness score such that strategies can be compared to one another and will be sorted according to this list. + +This is done to guide the genetic algorithm in its search through the space of strategies. For example, consider a population pool that has 4 empty strategies, and one strategy that breaks the TCP connection. After evaluation, the empty strategies would be killed off, and the strategy that runs is propagated. When the offspring of this strategy mutate, if one of them no longer breaks the underlying TCP connection, it will be considered more fit than the other strategies, and so on. + +This hierarchy accomplishes a significant *search space reduction*. Instead of Geneva fuzzing the entire space of possible strategies (for which there are many!), it instead quickly eliminates strategies that break the underlying connection and encourages the genetic algorithm to concentrate effort on only those strategies that keep the underlying connection alive. + +Said another way, this hierarchy allows Geneva to differentiate between a strategy shooting ourselves in the foot, and being caught by the censor. diff --git a/docs/howitworks/evaluator.rst b/docs/howitworks/evaluator.rst new file mode 100644 index 0000000..b38a061 --- /dev/null +++ b/docs/howitworks/evaluator.rst @@ -0,0 +1,268 @@ +Running the Evaluator +===================== + +Since Geneva only needs to run on one side of the connection, Geneva is flexible as to where and how it can be run. Strategy evaluation can be done either on the server-side or the client-side. When used on the server-side, it can drive an external client to itself (or another server) for testing, or even act as a NAT-ing middlebox to interpose between the external client and true server. + +The evaluator also allows plugins to override its default logic for single strategy evaluation, or for evaluating the entire strategy pool. See :ref:`Adding New Plugins` for more detail on writing your own plugin. + + +Client-side Evaluation +^^^^^^^^^^^^^^^^^^^^^^ + +The most common use case for Geneva is client-side evaluation. In this mode, +Geneva is run at the client-side inside a censored regime, trying to make a +query to a forbidden resource located outside the censored regime. For this +example, we will use the HTTP plugin. The HTTP plugin creates a forbidden HTTP +GET request (such as `example.com?q=ultrasurf` in China). + +The evaluator will start the engine, launch the client http plugin (which will +make a request), and then the client plugin will record if the request succeeded +or not. Under the hood, the GET request generated by the client HTTP plugin is +caught by the engine (which modifies the traffic according to the strategy it is +running) before sending it. + +This effectively looks like this: + +.. code-block:: none + + Censored Regime + ---------------------------------------------------------+ + | + Client - Geneva | + +---------------------------------------------+ | + | evaluator | | Server + | +-----------+ strategy | | +-----------------+ + | | http | GET /bad +----------+ | +----------+ | | + | | plugin <--------------> engine <-----------> censor <--------> forbidden.org | + | | | +----------+ | +----------+ | | + | +-----------+ | | +-----------------+ + | | | + +---------------------------------------------+ | + | + | + | + ---------------------------------------------------------+ + + +.. note:: For the purpose of this guide, examples will be given using :code:`evolve.py`. :code:`evolve.py` has an option :code:`--eval-only` to run the evaluator on a strategy with given parameters and exit. As usual, :code:`--eval-only` can also be replaced with parameters to the genetic algorithm to start the full genetic algorithm, instead of a single strategy evaluation. + +To accomplish this, we can simply run: + +.. code-block:: none + + # python3 evolve.py --eval-only "\/" --test-type http --server forbidden.org --log debug + +This will start :code:`evolve.py`, which will launch the `evaluator` with the +`http` plugin, configured to make a request to `forbidden.org`, in debug mode. + + +Server-side Evaluation +^^^^^^^^^^^^^^^^^^^^^^ + +Beyond using Geneva from the client-side, we can also use it from the server-side. + +In this mode, Geneva can learn strategies that work from the server-side, subverting censorship on client's behalf. +This requires a copy of Geneva to be checked out on both sides of the connection. To do this, we must specify the :code:`--server-side` flag. + +Geneva will first start up a server for the specified plugin, and start the engine with the given strategy in front of the server. It next SSHes into an external client worker located inside a censored regime and runs the specified plugin through the SSH session. +That plugin will generate a forbidden request to our server, and the plugin on the external client records if the request succeeded or not; the evaluator retrieves this fitness over the SSH session and evaluation is complete. This looks like this: + +.. code-block:: none + + Censored Regime + +------------------------------+ + | Server - Geneva + | +---------------------------------+ + Client | XXXXXXXXXX | | + +------------+ | XX XX | strategy +-----------+ | + | | GET /bad +----------+ X X | +--------+ | | | + | curl <-------------> censor <------> Internet <----------> engine <---> evaluator | | + | | +----------+ X X | +--------+ | | | + +-----^------+ | XX XX | +-----+-----+ | + | | XXXXXXXXXX | | | + | | +-----------------------+---------+ + | | | + +-----+------------------------+ | + | | + | 1. SSHs into the client and drives the request | + +----------------------------------------------------------------------------+ + +To run this with :code:`evolve.py` to evaluate an empty strategy (:code:`\/`), we can do: + +.. code-block:: none + + # python3 evolve.py --test-type http --public-ip --external-client example + --log debug --eval-only "\/" --server-side + +Note that we specified two additional options here: the public IP address of the test computer (used for making the request), and the external client worker we want to use. + +.. note:: See :ref:`Adding A Worker` on how to configure external workers for Geneva. + +External Clients with Local Servers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Alternatively, we can evaluate the strategies on the client side, but have Geneva host the server for us so we can observe the traffic on both sides of the connection. By simply not specifying the :code:`--server-side` flag, we can accomplish this. + +.. code-block:: none + + Censored Regime + +------------------------------------------+ + | Server + Geneva + | +-------------------+ + External Client | | | + +------------+ strategy | | +-----------+ | + | | GET /bad +--------+ +----+-----+ | | | | + | curl <------------> engine <----> censor <--------> evaluator | | + | | +--------+ +----+-----+ | | | | + +-----^------+ | | +-----+-----+ | + | | | | | + | | +-------------------+ + | | | + +------------------------------------------+ | + | | + | 1. SSHs into the client and drives the request | + +--------------------------------------------------------+ + + +.. code-block:: none + + # python3 evolve.py --test-type http --public-ip --external-client example + --log debug --eval-only "\/" --server-side + + + +External Clients with External Servers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Alternatively, the client can be driven to a different server from outside the censored regime with the :code:`'--external-server'` flag, and the server can be specified with :code:`'--server'`: + +.. code-block:: none + + # python3 evolve.py --test-type http --public-ip --external-client example + --log debug --eval-only "\/" --external-server --server http://wikipedia.org + +.. code-block:: none + + Geneva + +--------------------------+ + | | + | +-----------+ | + | | evaluator | | + | +-----+-----+ | + | | | + +-------------+------------+ + | + SSHes & drives request | + +----------------+ + | + | Censored Regime + +---------+----------------------------------------------+ + | | + | External Client | + +---------+-----------------------------------+ | + | | | | Server + | +-----v-----+ strategy | | +-----------------+ + | | http | GET /bad +----------+ | +-----+----+ | | + | | plugin <--------------> engine <-----------> censor <--------> forbidden.org | + | | | +----------+ | +-----+----+ | | + | +-----------+ | | +-----------------+ + | | | + +---------------------------------------------+ | + | + | + | + +--------------------------------------------------------+ + + +Engine as a Middlebox +~~~~~~~~~~~~~~~~~~~~~ + +There are many cases in which we cannot trigger a censor while communicating with servers we control. In these cases, +we can use Geneva as a middlebox, and interpose between the client and server and apply the strategy in between. + +To accomplish this, we will specify :code:`--act-as-middlebox`, and specify three additional routing options: + +Routing options: + - :code:`--routing-ip`: Internal IP address of the middlebox + - :code:`--forward-ip`: IP address we are forwarding to + - :code:`--sender-ip`: IP address we are forwarding from + +.. note:: Because Geneva operates at the packet-level, it cannot be used out of the box as a general purpose NAT, as it does not do connection tracking. + +In this mode, the external client will communicate directly with our IP address, which will in turn forward the packets to the specified destination after they are modified according to the strategy under evaluation. + +To run the strategy engine as a middlebox and drive an external client, we can add additional routing options: + +.. code-block:: none + + # python3 evolve.py --test-type http --public-ip --external-client example --log + debug --eval-only "\/" --server-side --act-as-middlebox --routing-ip + --forward-ip --sender-ip + +.. code-block:: none + + 1. SSHs into the client and drives the request + +------------------------------------------------+ + | | + | +---------------------+ + | Censored Regime | | | + +----------------------------+ | +-----+-----+ | + | | | | evaluator | | + v | | +-----------+ | + External Client | | | Server + +------------+ | | strategy | +---------+ + | | GET /bad +----+-----+ | +--------+ | | | + | curl <------------> censor <------------> engine <------------> bad.com | + | | +----+-----+ | +--------+ | | | + +------------+ | | | +---------+ + | +---------------------+ + | + +----------------------------+ + + +.. note:: The :code:`--routing-ip` is NOT the public IP address of your machine, its the internal address. This means that if you're running on an EC2 machine, the :code:`--public-ip` is your external IP, but the :code:`--routing-ip` is the IP you see when you run :code:`'ifconfig'`. + +Internal Evaluation with Docker +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For the purpose of internal testing and fitness function development, Geneva provides a set of simple mock censors (see :code:`censors/`). The evaluator can be configured to train against these censors. + +.. note:: In order to do internal evaluation against these mock censors, you must have set up Geneva's base Docker container. See :ref:`Setup` for how to do this. + +When used with Docker, Geneva will spin up three docker containers: a client, a +censor, and a server, and configure the networking routes such that the client +and server communicate through the censor. To evaluate strategies, Geneva will +run the plugin client inside the client and attempt to communicate with the +server through the censor. + +Each docker container used by the evaluator runs out of the same base container. + +.. code-block:: none + + evaluator + +--------------------------------------------------------------+ + | | + | Client Censor Server | + | +----------+ +----------+ +----------+ | + | | | | | | | | + | | client | | censor | | server | | + | | <-----+-----> <-----+-----> | | + | | plugin | ^ | | ^ | plugin | | + | | | | | | | | | | + | +----------+ | +----------+ | +----------+ | + | container | container | container | + +--------------------------------------------------------------+ + | | + | | + +------------------+ + engine can be run on either side + + +We can accomplish this by specifying the :code:`--censor ` option. This will enable running with Docker, start the censor, and perform evaluation. +Depending on whether :code:`--server-side` is specified, the engine will be run on either the client or server side. + +.. code-block:: none + + # python3 evolve.py --eval-only "" --test-type echo --censor censor2 --log debug + +.. note:: :code:`--censor` cannot be used with :code:`--external-client` or :code:`--external-server`. diff --git a/docs/howitworks/evolution.rst b/docs/howitworks/evolution.rst new file mode 100644 index 0000000..12a6785 --- /dev/null +++ b/docs/howitworks/evolution.rst @@ -0,0 +1,42 @@ +Evolution +========== + +Now that we have a concrete definition for a censorship evasion strategies and a way to use them, we can begin evolving new strategies with Geneva's genetic algorithm. + +A genetic algorithm is a type of machine learning inspired by natural selection. Over the course of many *generations*, it optimizes a numerical *fitness* score of individuals; within each generation, individuals that have a low fitness will not survive to the next generation, and those with a high fitness score will survive and propagate. + +Each individual in Geneva is a censorship evasion strategy. :code:`evolve.py` is the main driver for Geneva's genetic algorithm and maintains the population pool of these strategies. + +Each generation is comprised of the following: + 1. Mutation/Crossover - randomly mutating/mating strategies in the pool + 2. Strategy Evaluation - assigning a numerical fitness score to each strategy - decides which strategies should live on to the next generation + 3. Selection - the selection process of which strategies should survive to the next generation + +For more detail on mutation/crossover or the selection process, see `our papers `_ or the code in :code:`evolve.py` for more detail. :code:`evolve.py` exposes options to control hyperparameters of mutation/crossover, as well as other advanced options for rejecting mutations it has seen before. + +Strategy evaluation is significantly more complex, and will be covered in depth in the next section. + +For most of this documentation, we will show examples of using :code:`evolve.py` with the :code:`--eval-only ` option. This instructs :code:`evolve.py` not to start the full genetic algorithm, but to instead perform a single strategy evaluation with the given parameters. :code:`--eval-only` can be replaced with parameters to the genetic algorithm to start strategy evolution. + + +Argument Parsing +^^^^^^^^^^^^^^^^ + +Geneva uses a pass-through system of argument parsing. Different parts of the system define which arguments they care about, and they will parse just those args out. +If :code:`--help` is used, :code:`evolve.py` will collect the help messages for the relevant components (evaluator, plugins, etc). + + +Population Control +^^^^^^^^^^^^^^^^^^ + +The two most important command line options for controlling evolutions are :code:`--population` and +:code:`--generations`. These control the population size and number of generations evolution will +take place for, respectively. + +Geneva will not automatically halt evolution after population convergence occurs. + + +Seeding Population +^^^^^^^^^^^^^^^^^^ + +Geneva allows you to seed the starting population pool of strategies using the :code:`--seed ` parameter. This will create the initial population pool with nothing but copies of the given strategy. Mutation is applied to each individual before evaluation begins, so the first generation is not solely one individual being evaluated over and over again. diff --git a/docs/howitworks/howitworks.rst b/docs/howitworks/howitworks.rst new file mode 100644 index 0000000..2a44ed9 --- /dev/null +++ b/docs/howitworks/howitworks.rst @@ -0,0 +1,86 @@ +How it Works +============== + +See our `website `_ or our `research papers `_ for an in-depth read on how Geneva works. + +This documentation will provide a walkthrough of the main concepts behind Geneva, the main components of the codebase, and how they can be used. + +Censorship Evasion Strategies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A censorship evasion strategy is simply a *description of how network traffic should be modified*. A strategy is **not +code**, it is a description that tells Geneva's strategy engine how it should manipulate network traffic. + +The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted. + +A censorship evasion strategy composed of one or more packet-level building blocks. Geneva's core building blocks are: + +1. :code:`duplicate`: takes one packet and returns two copies of the packet +2. :code:`drop`: takes one packet and returns no packets (drops the packet) +3. :code:`tamper`: takes one packet and returns the modified packet +4. :code:`fragment`: takes one packet and returns two fragments or two segments + +Since :code:`duplicate` and :code:`fragment` introduce *branching*, these actions are composed into a binary-tree structure called an *action tree*. + +Each tree also has a *trigger*. The trigger describes which packets the tree should run on, and the tree describes what should happen to each of those packets when the trigger fires. Once a trigger fires on a packet, it pulls the packet into the tree for modifications, and the packets that emerge from the tree are sent on the wire. Recall that Geneva operates at the packet level, therefore all triggers are packet-level triggers. + +Multiple action trees together form a *forest*. Geneva handles outbound and inbound packets differently, so strategies are composed of two forests: an outbound forest and an inbound forest. + +Consider the following example of a simple Geneva strategy. + +.. code-block:: none + + +---------------+ triggers on TCP packets with the flags + | TCP:flags:A | <-- field set to 'ACK' - matching packets + +-------+-------+ are captured and pulled into the tree + | + +---------v---------+ makes two copies of the given packet. + duplicate <-- the tree is processed with an inorder + +---------+---------+ traversal, so the left side is run first + | + +-------------+------------+ + | | + +------------v----------+ v <-- dupilcate has no right child + tamper + {TCP:flags:replace:R} <-- parameters to this action describe how + +------------+----------+ the packet should be tampered + | + +------------v----------+ + tamper + {TCP:chksum:corrupt} + +------------+----------+ + | + v <-- packets that emerge from an in-order traversal + of the leaves are sent on the wire + +Strategy DNA Syntax +^^^^^^^^^^^^^^^^^^^ + +These strategies can be arbitrarily complicated, and Geneva defines a well-formatted string syntax for +unambiguously expressing strategies. + +A strategy divides how it handles outbound and inbound packets: these are separated in the DNA by a +"\\/". Specifically, the strategy format is :code:` \/ `. If :code:`\/` is not +present in a strategy, all of the action trees are in the outbound forest. + +Both forests are composed of action trees, and each forest is allowed an arbitrarily many trees. + +Action trees always start with a trigger, which is formatted as: :code:`[::]`. For example, the trigger: :code:`[TCP:flags:S]` will run its corresponding tree whenever it sees a :code:`TCP` packet with the :code:`flags` field set to :code:`SYN`. If the corresponding action tree is :code:`[TCP:flags:S]-drop-|`, this action tree will cause the engine to drop any :code:`SYN` packets. :code:`[TCP:flags:S]-duplicate-|` will cause the engine to duplicate any SYN packets. + +Triggers also can contain an optional 4th parameter for *gas*, which describes the number of times a trigger can fire. The triger :code:`[IP:version:4:4]` will run only on the first 4 IPv4 packets it sees. If the gas is negative, the trigger acts as a *bomb* trigger, which means the trigger will not fire until a certain number of applicable packets have been seen. For example, the trigger :code:`[IP:version:4:-2]` will trigger only after it has seen two matching packets (and it will not trigger on those first packets). + +Syntactically, action trees end with :code:`-|`. + +Depending on the type of action, some actions can have up to two children (such as :code:`duplicate`). These are represented +with the following syntax: :code:`[TCP:flags:S]-duplicate(,)-|`, where +:code:`` and :code:`` themselves are trees. If :code:`(,)` is not specified, any packets +that emerge from the action will be sent on the wire. If an action only has one child (such as :code:`tamper`), it is always the left child. :code:`[TCP:flags:S]-tamper{}(,)-|` + +Actions that have parameters specify those parameters within :code:`{}`. For example, giving parameters to the :code:`tamper` action could look like: :code:`[TCP:flags:S]-tamper{TCP:flags:replace:A}-|`. This strategy would trigger on TCP :code:`SYN` packets and replace the TCP :code:`flags` field to :code:`ACK`. + +Putting this all together, below is the strategy DNA representation of the above diagram: +:code:`[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/` + +Geneva has code to parse this strategy DNA into strategies that can be applied to network traffic using the engine. + +.. note:: Due to limitations of Scapy and NFQueue, actions that introduce branching (:code:`fragment`, :code:`duplicate`) are disabled for incoming action forests. diff --git a/docs/howitworks/logging.rst b/docs/howitworks/logging.rst new file mode 100644 index 0000000..1b0f75c --- /dev/null +++ b/docs/howitworks/logging.rst @@ -0,0 +1,28 @@ +Logging +======= + +Geneva uses multiple loggers during execution. :code:`evolve.py` creates the parent logger, and creates a subfolder of the current time under the :code:`trials/` directory. + +Within this directory, it creates 5 subfolders: + - :code:`data`: used for misc. data related to strategy evaluation + - :code:`flags`: used to write status files to set events + - :code:`generations`: used to store the full generations and hall of fame after each generation + - :code:`logs`: stores logs for evaluation + - :code:`packets`: stores packet captures during strategy evolution + +The two main log files used by :code:`evolve.py` are :code:`ga.log` and :code:`ga_debug.log` (everything in debug mode). As each strategy is evaluated, a :code:`_engine.log`, :code:`_server.log`, and :code:`_client.log` files are generated. + +For example, one run's output could be: + +.. code-block:: none + + # ls trials + 2020-03-23_20:03:08 + + # ls trials/2020-03-23_20:03:08 + data flags generations logs packets + + # ls trials/2020-03-23_20:03:08/logs + ga.log ga_debug.log zhak1n81_client.log zhak1n81_engine.log zhak1n81_server.log + + diff --git a/docs/howitworks/puttingittogether.rst b/docs/howitworks/puttingittogether.rst new file mode 100644 index 0000000..3e41254 --- /dev/null +++ b/docs/howitworks/puttingittogether.rst @@ -0,0 +1,21 @@ +Putting it all Together +======================= + +Now that we know how to leverage the framework, this short section will give a high level of +how to put it all together and run Geneva's genetic algorithm yourself. + +The two most important command line options for controlling evolutions are :code:`--population` and +:code:`--generations`. These control the population size and number of generations evolution will +take place for, respectively. + +Generally, these need not be very large. Geneva intentionally does strategy evaluation serially & slowly, so increasing the size dramatically will make evolution take longer. For our existing research papers, the population count rarely exceeded 300. + +For example, to run a client-side evolution against HTTP censorship with a population pool of 200 and 25 generations, the following can be used: + +.. code-block:: none + + # python3 evolve.py --population 200 --generations 25 --test-type http --server forbidden.org + +Before running any evolution, it is recommended to spot test the plugins against whatever censor is used as the adversary to confirm the fitness function properly sets up the desirable hierarchy of individuals as specified in :ref:`Fitness Functions`: strategies that break the connection get lower fitness than those that get censored, which get a lower fitness than those that succeed. + +And that's it! diff --git a/docs/howitworks/testing.rst b/docs/howitworks/testing.rst new file mode 100644 index 0000000..2f5f218 --- /dev/null +++ b/docs/howitworks/testing.rst @@ -0,0 +1,13 @@ +Automated Tests +=============== + +Geneva has a system of automated tests in the :code:`tests/` directory, powered by pytest. Unless you are doing +modifications to the source code, you can generally ignore these. + +If you need to run them yourself, you can do so with: + +.. code-block:: none + + # python3 -m pytest -sv tests/ + +To put the tests in debug mode, you can add :code:`--evolve-logger debug`. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..1469440 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,45 @@ +.. geneva documentation master file, created by + sphinx-quickstart on Fri Apr 10 12:35:13 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Geneva's documentation! +================================== + + +**Disclaimer:** Running Geneva or Geneva's strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it. + +.. toctree:: + :caption: Getting Started: + + intro/introduction + intro/setup + intro/gettingstarted + +.. toctree:: + :caption: Usage: + + howitworks/howitworks + howitworks/engine + howitworks/evolution + howitworks/evaluation + howitworks/evaluator + howitworks/addingaworker + howitworks/logging + howitworks/testing + howitworks/puttingittogether + +.. toctree:: + :caption: Extending Geneva: + + extending/plugins + extending/actions + extending/contributing + +.. toctree:: + :glob: + :caption: API Reference: + + api/* + api/actions/* + api/plugins/* diff --git a/docs/intro/gettingstarted.rst b/docs/intro/gettingstarted.rst new file mode 100644 index 0000000..bbee4a5 --- /dev/null +++ b/docs/intro/gettingstarted.rst @@ -0,0 +1,93 @@ +Getting Started +================= + +See our `website `_ or our `research papers `_ for an in-depth read on how Geneva works. + +This documentation will provide a walkthrough of the main concepts behind Geneva, the main components of the codebase, and how they can be used. + +This section will give a high level overview on how Geneva works; before using it, you are **strongly recommended** to read through :ref:`How it Works`. + + +What is a Strategy? +~~~~~~~~~~~~~~~~~~~ + +A censorship evasion strategy is simply a *description of how network traffic +should be modified*. A strategy is **not code**, it is a description that tells the +*strategy engine* how it should manipulate network traffic. + +The goal of a censorship evasion strategy is to modify the network traffic in a such a way that the censor is unable to censor it, but the client/server communication is unimpacted. + +Strategies & Species +~~~~~~~~~~~~~~~~~~~~ + +Because Geneva commonly identifies many different strategies, we have defined a +*taxonomy* to classify strategies into. + +The Strategy taxonomy is as follows, ordered from most general to most specific: + - 1. Species: The overarching bug a strategy exploits + - 2. Subspecies: The mechanism used to exploit the bug + - 3. Variant: Salient wireline differences using the same bug mechanism + +The highest level classification is *species*, +a broad class of strategies classified by the type of weakness it exploits in a +censor implementation. :code:`TCB Teardown` is an example of one such species; if the +censor did not prematurely teardown TCBs, all the strategies in this species +would cease to function. + +Within each species, different *subspecies* represent +unique ways to exploit the weakness that defines the strategy. For example, +injecting an insertion TCP :code:`RST` packet would comprise one subspecies within the +TCB Teardown species; injecting a TCP :code:`FIN` would comprise another. + +Within each +subspecies, we further record *variants*, unique strategies that leverage the same +attack vector, but do so slightly differently: corrupting the checksum field on +a :code:`RST` packet is one variant of the :code:`TCB Teardown w/ RST` subspecies of the :code:`TCB Teardown` species; +corrupting the :code:`ack` field is another. + +We refer to specific individuals as *extinct* +if they once worked against a censor but are no longer effective (less than 5% +success rate). That formerly successful approaches could, after a few years, become +ineffective lends further motivation for a technique that can quickly learn new +strategies. + + +Running a Strategy +~~~~~~~~~~~~~~~~~~ + +For a fuller description of the DNA syntax, see :ref:`Censorship Evasion Strategies`. + +.. code-block:: bash + + # python3 engine.py --server-port 80 --strategy "\/" --log debug + 2019-10-14 16:34:45 DEBUG:[ENGINE] Engine created with strategy \/ (ID bm3kdw3r) to port 80 + 2019-10-14 16:34:45 DEBUG:[ENGINE] Configuring iptables rules + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p tcp --sport 80 -j NFQUEUE --queue-num 1 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p tcp --dport 80 -j NFQUEUE --queue-num 2 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A OUTPUT -p udp --sport 80 -j NFQUEUE --queue-num 1 + 2019-10-14 16:34:45 DEBUG:[ENGINE] iptables -A INPUT -p udp --dport 80 -j NFQUEUE --queue-num 2 + + +Note that if you have stale :code:`iptables` rules or other rules that rely on Geneva's default queues, +this will fail. To fix this, remove those rules. + +Strategy Library +~~~~~~~~~~~~~~~~~ + +Geneva has found dozens of strategies that work against censors in China, +Kazakhstan, India, and Iran. We include several of these strategies in +`strategies.md `_ . Note that this file contains success rates for +each individual country; a strategy that works in one country may not work as +well as other countries. + +Researchers have observed that strategies may have differing success rates based +on your exact location. Although we have not observed this from our vantage +points, you may find that some strategies may work differently in a country we +have tested. If this is the case, don't be alarmed. However, please feel free to +reach out to a member of the team directly or open an issue on this page so we +can track how the strategies work from other geographic locations. + +Disclaimer +========== + +Running these strategies may place you at risk if you use it within a censoring regime. Geneva takes overt actions that interfere with the normal operations of a censor and its strategies are detectable on the network. During the training process, Geneva will intentionally trip censorship many times. Geneva is not an anonymity tool, nor does it encrypt any traffic. Understand the risks of running Geneva in your country before trying it. diff --git a/docs/intro/introduction.rst b/docs/intro/introduction.rst new file mode 100644 index 0000000..b8b1f0c --- /dev/null +++ b/docs/intro/introduction.rst @@ -0,0 +1,28 @@ +Introduction +=================================== + +Geneva is an artificial intelligence tool that defeats censorship by exploiting +bugs in censors, such as those in China, India, and Kazakhstan. Unlike many +other anti-censorship solutions which require assistance from outside the +censoring regime (Tor, VPNs, etc.), Geneva runs strictly on one side of the +connection (either the client or server side). + +Under the hood, Geneva uses a genetic algorithm to evolve censorship evasion +strategies and has found several previously unknown bugs in censors. Geneva's +strategies manipulate the network stream to confuse the censor without impacting +the client/server communication. This makes Geneva effective against many types +of in-network censorship (though it cannot be used against IP-blocking +censorship). + +Geneva is composed of two high level components: its *genetic algorithm* (which +it uses to evolve new censorship evasion strategies) and its *strategy engine* +(which is uses to run an individual censorship evasion strategy over a network +connection). + +Geneva's `Github page `_ contains the +Geneva's full implementation: its genetic algorithm, strategy engine, Python +API, and a subset of published strategies. With these tools, users and +researchers alike can evolve new strategies or leverage existing strategies to +evade censorship. To learn more about how Geneva works, see :ref:`How it +Works`. + diff --git a/docs/intro/setup.rst b/docs/intro/setup.rst new file mode 100644 index 0000000..d6042ab --- /dev/null +++ b/docs/intro/setup.rst @@ -0,0 +1,49 @@ +Setup +===== + +Geneva has been developed and tested for Centos or Debian-based systems. Due to +limitations of netfilter and raw sockets, Geneva does not work on OS X or +Windows at this time and requires *python3.6*. + +Install netfilterqueue dependencies: + +.. code-block:: bash + + # sudo apt-get install build-essential python-dev libnetfilter-queue-dev libffi-dev libssl-dev iptables python3-pip + + +Install Python dependencies: + +.. code-block:: bash + + # python3 -m pip install -r requirements.txt + +Docker (Optional) +^^^^^^^^^^^^^^^^^ + +Geneva has an internal system that can be used to test strategies using Docker. +This is largely used for testing fitness functions with the mock censors +provided - it is **not used for training against real censors**. Due to +limitations of raw sockets inside docker containers in many builds of Docker, +Geneva cannot be used inside a docker container to communicate with hosts +outside of Docker's internal network. + +When used with Docker, Geneva will spin up three docker containers: a client, a +censor, and a server, and configure the networking routes such that the client +and server communicate through the censor. To evaluate strategies (see much more detail in the evaluation section), +Geneva will run the plugin client inside the client and +attempt to communicate with the server through the censor. + +Each docker container used by the evaluator runs out of the same base container. + +Build the base container with: + +.. code-block:: bash + + docker build -t base:latest -f docker/Dockerfile . + +Optionally, to manually run/inspect the docker image to explore the image, run: + +.. code-block:: bash + + docker run -it base diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..2119f51 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/engine.py b/engine.py index a5c5797..d4dce2d 100644 --- a/engine.py +++ b/engine.py @@ -1,8 +1,8 @@ """ -Engine +Geneva Strategy Engine -Given a strategy and a server port, the engine configures NFQueue -so the strategy can run on the underlying connection. +Given a strategy and a server port, the engine configures NFQueue to capture all traffic +into and out of that port so the strategy can run over the connection. """ import argparse @@ -14,7 +14,10 @@ import subprocess import threading import time -import netfilterqueue +try: + import netfilterqueue +except ImportError: + pass from scapy.layers.inet import IP from scapy.utils import wrpcap @@ -30,28 +33,75 @@ BASEPATH = os.path.dirname(os.path.abspath(__file__)) class Engine(): - def __init__(self, server_port, string_strategy, server_side=False, environment_id=None, output_directory="trials", log_level="info"): + def __init__(self, server_port, + string_strategy, + environment_id=None, + server_side=False, + output_directory="trials", + log_level="info", + enabled=True, + in_queue_num=None, + out_queue_num=None, + forwarder=None, + save_seen_packets=True): + """ + Args: + server_port (int): The port the engine will monitor + string_strategy (str): String representation of strategy DNA to apply to the network + environment_id (str, None): ID of the given strategy + server_side (bool, False): Whether or not the engine is running on the server side of the connection + output_directory (str, 'trials'): The path logs and packet captures should be written to + enabled (bool, True): whether or not the engine should be started (used for conditional context managers) + in_queue_num (int, None): override the netfilterqueue number used for inbound packets. Used for running multiple instances of the engine at the same time. Defaults to None. + out_queue_num (int, None): override the netfilterqueue number used for outbound packets. Used for running multiple instances of the engine at the same time. Defaults to None. + save_seen_packets (bool, True): whether or not the engine should record and save packets it sees while running. Defaults to True, but it is recommended this be disabled on higher throughput systems. + """ self.server_port = server_port + # whether the engine is running on the server or client side. + # this affects which direction each out/in tree is attached to the + # source and destination port. + self.server_side = server_side + self.overhead = 0 self.seen_packets = [] - # Set up the directory and ID for logging - actions.utils.setup_dirs(output_directory) - if not environment_id: - environment_id = actions.utils.get_id() - self.environment_id = environment_id + self.forwarder = forwarder + self.save_seen_packets = save_seen_packets + if forwarder: + self.sender_ip = forwarder["sender_ip"] + self.routing_ip = forwarder["routing_ip"] + self.forward_ip = forwarder["forward_ip"] + + # Set up the directory and ID for logging + if not output_directory: + self.output_directory = "trials" + else: + self.output_directory = output_directory + actions.utils.setup_dirs(self.output_directory) + if not environment_id: + self.environment_id = actions.utils.get_id() + # Set up a logger self.logger = actions.utils.get_logger(BASEPATH, - output_directory, + self.output_directory, __name__, "engine", - environment_id, + self.environment_id, log_level=log_level) - self.output_directory = output_directory - self.server_side = server_side + # Warn if these are not provided + if not environment_id: + self.logger.warning("No environment ID given, one has been generated (%s)", self.environment_id) + if not output_directory: + self.logger.warning("No output directory specified, using the default (%s)" % self.output_directory) # Used for conditional context manager usage + self.enabled = enabled + + # Parse the given strategy self.strategy = actions.utils.parse(string_strategy, self.logger) + # Setup variables used by the NFQueue system + self.in_queue_num = in_queue_num or 1 + self.out_queue_num = out_queue_num or self.in_queue_num + 1 self.out_nfqueue_started = False self.in_nfqueue_started = False self.running_nfqueue = False @@ -71,22 +121,51 @@ class Engine(): def __enter__(self): """ Allows the engine to be used as a context manager; simply launches the - engine. + engine if enabled. """ - self.initialize_nfqueue() + if self.enabled: + self.initialize_nfqueue() return self def __exit__(self, exc_type, exc_value, tb): """ Allows the engine to be used as a context manager; simply stops the engine + if enabled. """ - self.shutdown_nfqueue() + if self.enabled: + self.shutdown_nfqueue() + + def do_nat(self, packet): + """ + NATs packet: changes the sources and destination IP if it matches the + configured route, and clears the checksums for recalculating + + Args: + packet (Actions.packet.Packet): packet to modify before sending + + Returns: + Actions.packet.Packet: the modified packet + """ + if packet["IP"].src == self.sender_ip: + packet["IP"].dst = self.forward_ip + packet["IP"].src = self.routing_ip + del packet["TCP"].chksum + del packet["IP"].chksum + elif packet["IP"].src == self.forward_ip: + packet["IP"].dst = self.sender_ip + packet["IP"].src = self.routing_ip + del packet["TCP"].chksum + del packet["IP"].chksum + return packet def mysend(self, packet): """ Helper scapy sending method. Expects a Geneva Packet input. """ try: + if self.forwarder: + self.logger.debug("NAT-ing packet.") + packet = self.do_nat(packet) self.logger.debug("Sending packet %s", str(packet)) self.socket.send(packet.packet) except Exception: @@ -113,7 +192,8 @@ class Engine(): self.in_nfqueue_started = True nfqueue.run_socket(nfqueue_socket) - except socket.timeout: + # run_socket can raise an OSError on shutdown for some builds of netfilterqueue + except (socket.timeout, OSError): pass except Exception: self.logger.exception("Exception out of run_nfqueue() (direction=%s)", direction) @@ -124,9 +204,10 @@ class Engine(): """ self.logger.debug("Configuring iptables rules") - port1, port2 = "dport", "sport" - if self.server_side: - port1, port2 = "sport", "dport" + # Switch source and destination ports if this evaluator is to run from the server side + port1, port2 = "sport", "dport" + if not self.server_side: + port1, port2 = "dport", "sport" out_chain = "OUTPUT" in_chain = "INPUT" @@ -137,16 +218,23 @@ class Engine(): add_or_remove = "D" cmds = [] for proto in ["tcp", "udp"]: - cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 1" % - (add_or_remove, out_chain, proto, port1, self.server_port), - "iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num 2" % - (add_or_remove, in_chain, proto, port2, self.server_port)] + cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" % + (add_or_remove, out_chain, proto, port1, self.server_port, self.out_queue_num), + "iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" % + (add_or_remove, in_chain, proto, port2, self.server_port, self.in_queue_num)] + # If this machine is acting as a middlebox, we need to add the same rules again + # in the opposite direction so that we can pass packets back and forth + if self.forwarder: + cmds += ["iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" % + (add_or_remove, out_chain, proto, port2, self.server_port, self.out_queue_num), + "iptables -%s %s -p %s --%s %d -j NFQUEUE --queue-num %d" % + (add_or_remove, in_chain, proto, port1, self.server_port, self.in_queue_num)] for cmd in cmds: self.logger.debug(cmd) - # If we're logging at DEBUG mode, keep stderr/stdout piped to us + # If we're logging at debug mode, keep stderr/stdout piped to us # Otherwise, pipe them both to DEVNULL - if actions.utils.get_console_log_level() == logging.DEBUG: + if actions.utils.get_console_log_level() == "debug": subprocess.check_call(cmd.split(), timeout=60) else: subprocess.check_call(cmd.split(), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) @@ -167,8 +255,8 @@ class Engine(): self.out_nfqueue = netfilterqueue.NetfilterQueue() self.in_nfqueue = netfilterqueue.NetfilterQueue() # Bind them - self.out_nfqueue.bind(1, self.out_callback) - self.in_nfqueue.bind(2, self.in_callback) + self.out_nfqueue.bind(self.out_queue_num, self.out_callback) + self.in_nfqueue.bind(self.in_queue_num, self.in_callback) # Create our nfqueue sockets to allow for non-blocking usage self.out_nfqueue_socket = socket.fromfd(self.out_nfqueue.get_fd(), socket.AF_UNIX, @@ -211,6 +299,9 @@ class Engine(): if self.out_nfqueue: self.out_nfqueue.unbind() self.configure_iptables(remove=True) + self.socket.close() + self.out_nfqueue_socket.close() + self.in_nfqueue_socket.close() packets_path = os.path.join(BASEPATH, self.output_directory, @@ -218,7 +309,8 @@ class Engine(): "original_%s.pcap" % self.environment_id) # Write to disk the original packets we captured - wrpcap(packets_path, [p.packet for p in self.seen_packets]) + if self.save_seen_packets: + wrpcap(packets_path, [p.packet for p in self.seen_packets]) # If the engine exits before it initializes for any reason, these threads may not be set # Only join them if they are defined @@ -241,7 +333,8 @@ class Engine(): self.logger.debug("Received outbound packet %s", str(packet)) # Record this packet for a .pacp later - self.seen_packets.append(packet) + if self.save_seen_packets: + self.seen_packets.append(packet) # Drop the packet in NFQueue so the strategy can handle it nfpacket.drop() @@ -253,6 +346,8 @@ class Engine(): Handles processing an outbound packet through the engine. """ packets_to_send = self.strategy.act_on_packet(packet, self.logger, direction="out") + if packets_to_send: + self.overhead += (len(packets_to_send) - 1) # Send all of the packets we've collected to send for out_packet in packets_to_send: @@ -272,14 +367,15 @@ class Engine(): return packet = actions.packet.Packet(IP(nfpacket.get_payload())) - self.seen_packets.append(packet) + if self.save_seen_packets: + self.seen_packets.append(packet) self.logger.debug("Received packet: %s", str(packet)) # Run the given strategy packets = self.strategy.act_on_packet(packet, self.logger, direction="in") - # Censors will often send RA packets to disrupt a TCP stream - record this + # GFW will send RA packets to disrupt a TCP stream if packet.haslayer("TCP") and packet.get("TCP", "flags") == "RA": self.censorship_detected = True @@ -289,6 +385,11 @@ class Engine(): nfpacket.drop() return + if self.forwarder: + nfpacket.drop() + self.handle_packet(packet) + return + # Otherwise, overwrite this packet with the packet the action trees gave back nfpacket.set_payload(bytes(packets[0])) @@ -306,13 +407,20 @@ def get_args(): """ parser = argparse.ArgumentParser(description='The engine that runs a given strategy.') parser.add_argument('--server-port', type=int, action='store', required=True) - parser.add_argument('--server-side', action='store_true', help="If this strategy is running on the server side of a connection") - parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test. If not provided, one will be generated.") + parser.add_argument('--environment-id', action='store', help="ID of the current strategy under test") + parser.add_argument('--sender-ip', action='store', help="IP address of sending machine, used for NAT") + parser.add_argument('--routing-ip', action='store', help="Public IP of this machine, used for NAT") + parser.add_argument('--forward-ip', action='store', help="IP address to forward traffic to") parser.add_argument('--strategy', action='store', help="Strategy to deploy") parser.add_argument('--output-directory', default="trials", action='store', help="Where to output logs, captures, and results. Defaults to trials/.") + parser.add_argument('--forward', action='store_true', help='Enable if this is forwarding traffic') + parser.add_argument('--server-side', action='store_true', help='Enable if this is running on the server side') parser.add_argument('--log', action='store', default="debug", choices=("debug", "info", "warning", "critical", "error"), help="Sets the log level") + parser.add_argument('--no-save-packets', action='store_false', help='Disables recording captured packets') + parser.add_argument("--in-queue-num", action="store", help="NfQueue number for incoming packets", default=1, type=int) + parser.add_argument("--out-queue-num", action="store", help="NfQueue number for outgoing packets", default=None, type=int) args = parser.parse_args() return args @@ -323,12 +431,22 @@ def main(args): Kicks off the engine with the given arguments. """ try: + nat_config = {} + if args.get("sender_ip") and args.get("routing_ip") and args.get("forward_ip"): + nat_config = {"sender_ip" : args["sender_ip"], + "routing_ip" : args["routing_ip"], + "forward_ip" : args["forward_ip"]} + eng = Engine(args["server_port"], args["strategy"], - environment_id=args.get("environment_id"), - output_directory = args.get("output_directory"), - server_side=args.get("server_side"), - log_level=args["log"]) + environment_id=args["environment_id"], + server_side=args["server_side"], + output_directory=args["output_directory"], + forwarder=nat_config, + log_level=args["log"], + in_queue_num=args["in_queue_num"], + out_queue_num=args["out_queue_num"], + save_seen_packets=args["no-save-packets"]) eng.initialize_nfqueue() while True: time.sleep(0.5) diff --git a/evaluator.py b/evaluator.py new file mode 100644 index 0000000..0976add --- /dev/null +++ b/evaluator.py @@ -0,0 +1,1156 @@ +""" +The Evaluator is charged with evaluating a given strategy and assigning a numerical fitness metric to it. +""" +import argparse +import copy +import logging +import multiprocessing +import os +import random +import socket +import subprocess +import sys +import threading +import time +import re +import warnings + +import requests +import urllib3 + +import actions.utils +import censors.censor_driver + + +# Suppress unfixed Paramiko warnings (see Paramiko issue #1386) +warnings.filterwarnings(action='ignore',module='.*paramiko.*') + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = BASEPATH + + +class Evaluator(): + def __init__(self, command, logger): + """ + Initialize the global evaluator for this evolution. + + Args: + command (list): sys.argv or list of arguments + logger (:obj:`logging.Logger`): logger passed in from the main driver to log from + """ + self.args = get_args(command) + self.test_plugin = self.args["test_type"] + assert self.test_plugin, "Cannot import an empty plugin" + + self.public_ip = self.args.get("public_ip", "") + + self.external_client = self.args["external_client"] + self.censor = self.args.get("censor") + # If there is no external client defined and no internal test setup, default --external-server to True + if not self.external_client and not self.censor: + self.args["external_server"] = True + + self.external_server = self.args["external_server"] + + # If there is an external client connecting to us, override the server with our public ip + if not self.external_server and self.external_client: + assert self.args.get("public_ip", ""), "Cannot use an external client to this server without specifying the public IP." + self.public_ip = self.args.get("public_ip", "") + worker = actions.utils.get_worker(self.public_ip, logger) + if worker: + self.public_ip = worker["ip"] + self.args.update({'server': self.public_ip}) + command += ["--server", self.public_ip] + + self.run_canary_phase = True + + self.client_args = copy.deepcopy(self.args) + self.server_args = copy.deepcopy(self.args) + + self.client_cls = None + self.server_cls = None + self.plugin = None + + self.override_evaluation = False + + # Plugin may optionally override the strategy evaluation for a single individual or the entire evaluation + try: + _, plugin_cls = actions.utils.import_plugin(self.test_plugin, "plugin") + parsed_args = plugin_cls.get_args(command) + self.args.update({k:v for k,v in parsed_args.items() if v or (not v and k not in self.args)}) + self.plugin = plugin_cls(self.args) + # Disable the canary phase if the plugin will override the default evaluation logic + self.run_canary_phase = not self.plugin.override_evaluation + self.override_evaluation = self.plugin.override_evaluation + except ImportError: + pass + + self.client_cls = collect_plugin(self.test_plugin, "client", command, self.args, self.client_args) + self.server_cls = collect_plugin(self.test_plugin, "server", command, self.args, self.server_args) + + self.workers = self.args["workers"] + self.stop = False + self.skip_empty = not self.args["no_skip_empty"] + self.output_directory = self.args["output_directory"] + + self.routing_ip = self.args.get("routing_ip", None) + self.runs = self.args.get("runs", 1) + self.fitness_by = self.args.get("fitness_by", "avg") + + self.forwarder = {} + # If NAT options were specified to train as a middle box, set up the engine's + # NAT configuration + self.act_as_middlebox = self.args.get("act_as_middlebox") + if self.act_as_middlebox: + assert self.args.get("forward_ip") + assert self.args.get("sender_ip") + assert self.args.get("routing_ip") + self.forwarder["forward_ip"] = self.args["forward_ip"] + self.forwarder["sender_ip"] = self.args["sender_ip"] + self.forwarder["routing_ip"] = self.args["routing_ip"] + + # Legacy environments storage + self.environments = [] + if not os.path.exists(self.output_directory): + os.mkdir(self.output_directory) + + # Only enable docker if we're going to use an internal censor + self.use_docker = False + if self.args["censor"]: + import docker + self.use_docker = True + self.docker_client = docker.from_env() + self.apiclient = docker.APIClient() + + self.logger = logger + + def evaluate(self, ind_list): + """ + Perform the overall fitness evaluation driving. + + Args: + ind_list (list): list of individuals to evaluate + + Returns: + list: Population list after evaluation + """ + # Setup environment ids for each individual + self.assign_ids(ind_list) + + # If the plugin has overridden default evaluation, call that here + if self.override_evaluation: + self.logger.debug("Beginning evaluation in plugin") + return self.plugin.evaluate(self.args, self, ind_list, self.logger) + + if self.workers > 1 and self.use_docker: + # Chunk the population and test sites into smaller lists to hand to each worker + split = [ind_list[i::self.workers] for i in range(0, self.workers)] + + procs = [] + # Create workers + for i in range(0, len(split)): + if not split[i]: + continue + if self.use_docker: + try: + # Due to limitations in docker-py, it is not safe to build the containers in a multiprocessed + # setting. To handle this, build the environments ahead of time, and pass them to the workers to use. + environment = self.create_test_environment(i) + except (docker.errors.APIError, requests.exceptions.ConnectionError, urllib3.exceptions.ProtocolError): + self.logger.exception("Failed to create evaluator environment - is docker running?") + return + + proc = multiprocessing.Process(target=self.worker, args=(split[i], str(i), environment)) + proc.start() + procs.append(proc) + + try: + # Join all the processes + for proc in procs: + proc.join() + except KeyboardInterrupt: + self.shutdown() + else: + environment = {} + if self.use_docker: + try: + environment = self.create_test_environment("main") + except (docker.errors.APIError, requests.exceptions.ConnectionError, urllib3.exceptions.ProtocolError): + self.logger.exception("Failed to create evaluator environment - is docker running?") + return + + self.worker(ind_list, "main", environment) + + for ind in ind_list: + self.read_fitness(ind) + self.terminate_docker() + + return ind_list + + def run_test(self, environment, ind): + """ + Conducts a test of a given individual in the environment. + + Args: + environment (dict): Dictionary of environment variables + ind (:obj:`actions.strategy.Strategy`): A strategy object to test with + + Returns: + tuple: (ind.environment_id, ind.fitness) environment ID of strategy and fitness + """ + # If skip_empty is enabled, this is not the canary, and the individual is empty, + # skip it + if len(ind) == 0 and ind.environment_id != "canary" and self.skip_empty: + self.logger.info("[skipped] Fitness %d: %s" % (-1000, str(ind))) + ind.fitness = -1000 + return "skipped", -1000 + + fitnesses = [] + + # Run the strategy multiple times if requested + for run in range(0, self.runs): + self.logger.debug("Launching %s plugin (run %d) for %s" % (self.test_plugin, run + 1, str(ind))) + + environment["id"] = ind.environment_id + self.client_args.update({"environment_id": ind.environment_id}) + self.server_args.update({"environment_id": ind.environment_id}) + + if not self.args["server_side"]: + self.client_args.update({"strategy" : str(ind)}) + self.server_args.update({"no_engine" : True}) + else: + self.server_args.update({"strategy" : str(ind)}) + self.client_args.update({"no_engine" : True}) + + # If we're using an internal censor, make sure the client is pointed at the server + if self.args["censor"]: + self.client_args.update({"server": environment["server"]["ip"]}) + self.client_args.update({"wait_for_censor": True}) + self.server_args.update({"wait_for_shutdown": True}) + self.update_ports(environment) + + try: + # If the plugin has overridden the below logic, run that plugin's version directly + if self.plugin: + self.logger.debug("Running standalone plugin.") + self.args.update({"strategy": str(ind)}) + self.plugin.start(self.args, self, environment, ind, self.logger) + self.read_fitness(ind) + else: + self.logger.debug("Launching client and server directly.") + # If we're given a server to start, start it now + if self.server_cls and not self.external_server and not self.act_as_middlebox: + server = self.start_server(self.server_args, environment, self.logger) + + fitness = self.run_client(self.client_args, environment, self.logger) + + if self.server_cls and not self.external_server and not self.act_as_middlebox: + self.stop_server(environment, server) + + self.read_fitness(ind) + + # If the engine ran on the server side, ask that it punish fitness + if self.args["server_side"]: + ind.fitness = server.punish_fitness(ind.fitness, self.logger) + actions.utils.write_fitness(ind.fitness, self.output_directory, environment["id"]) + except actions.utils.SkipStrategyException as exc: + self.logger.debug("Strategy evaluation ending.") + ind.fitness = exc.fitness + fitnesses.append(ind.fitness) + break + + fitnesses.append(ind.fitness) + + if self.runs > 1: + self.logger.debug("\t(%d/%d) Fitness %s: %s" % (run + 1, self.runs, str(ind.fitness), str(ind))) + + self.logger.debug("Storing fitness of %s by: %s" % (fitnesses, self.fitness_by)) + if self.fitness_by == "min": + ind.fitness = min(fitnesses) + elif self.fitness_by == "max": + ind.fitness = max(fitnesses) + elif self.fitness_by == "avg": + ind.fitness = round(sum(fitnesses)/len(fitnesses), 2) + actions.utils.write_fitness(ind.fitness, self.output_directory, environment["id"]) + + # Log the fitness + self.logger.info("[%s] Fitness %s: %s" % (ind.environment_id, str(ind.fitness), str(ind))) + + return ind.environment_id, ind.fitness + + def run_client(self, args, environment, logger): + """ + Runs the plugin client given the current configuration + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + + Returns: + float: Fitness of individual + """ + fitness = None + if environment.get("remote"): + fitness = self.run_remote_client(args, environment, logger) + elif environment.get("docker"): + self.run_docker_client(args, environment, logger) + else: + self.run_local_client(args, environment, logger) + + fitpath = os.path.join(BASEPATH, self.output_directory, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness" + # Do not overwrite the fitness if it already exists + if not os.path.exists(fitpath): + actions.utils.write_fitness(fitness, self.output_directory, environment["id"]) + return fitness + + def run_docker_client(self, args, environment, logger): + """ + Runs client within the docker container. Does not return fitness; instead + fitness is written via the flags directory and read back in later. + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + """ + command = ["docker", "exec", "--privileged", environment["client"]["container"].name, "python", "code/plugins/plugin_client.py", "--server", environment["server"]["ip"]] + base_cmd = actions.utils.build_command(args) + command += base_cmd + self.exec_cmd(command) + + def update_ports(self, environment): + """ + Checks that the chosen port is open inside the docker container - if not, it chooses a new port. + + Args: + environment (dict): Dictionary describing docker environment + """ + command = ["docker", "exec", "--privileged", environment["server"]["container"].name, "netstat", "-ano"] + output = self.exec_cmd_output(command) + requested_port = self.args.get("port") + self.logger.debug("Testing if port %s is open in the docker container" % requested_port) + while (":%s" % requested_port) in output: + self.logger.warn("Port %s is in use, choosing a new port" % requested_port) + requested_port = random.randint(1000, 65000) + output = self.exec_cmd_output(command) + self.logger.debug("Using port %s" % requested_port) + self.args.update({"port": requested_port}) + self.client_args.update({"port": requested_port}) + self.server_args.update({"port": requested_port}) + + def run_remote_client(self, args, environment, logger): + """ + Runs client remotely over SSH, using the given SSH channel + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + + Returns: + float: Fitness of individual + """ + worker = environment["worker"] + remote = environment["remote"] + command = [] + if worker["username"] != "root": + command = ["sudo"] + command += [worker["python"], os.path.join(worker["geneva_path"], "plugins/plugin_client.py")] + base_cmd = actions.utils.build_command(args) + command += base_cmd + command = " ".join(command) + + self.remote_exec_cmd(remote, command, logger, timeout=20) + + # Get the logs from the run + self.get_log(remote, worker, "%s.client.log" % environment["id"], logger) + if not args["server_side"]: + self.get_log(remote, worker, "%s.engine.log" % environment["id"], logger) + + # Get the individual's fitness + command = 'cat %s/%s/%s/%s.fitness' % (worker["geneva_path"], self.output_directory, actions.utils.FLAGFOLDER, environment["id"]) + remote_fitness, error_lines = self.remote_exec_cmd(remote, command, logger) + fitness = -1000 + try: + fitness = int(remote_fitness[0]) + except Exception: + logger.exception("Failed to parse remote fitness.") + return None + + return fitness + + def remote_exec_cmd(self, remote, command, logger, timeout=15, verbose=True): + """ + Given a remote SSH session, executes a string command. Blocks until + command completes, and returns the stdout and stderr. If the SSH + connection is broken, it will try again. + + Args: + remote: Paramiko SSH channel to execute commands over + command (str): Command to execute + logger (:obj:`logging.Logger`): A logger to log with + timeout (int, optional): Timeout for the command + verbose (bool, optional): Whether the output should be printed + + Returns: + tuple: (stdout, stderr) of command, each is a list + """ + i, max_tries = 0, 3 + lines = [] + error_lines = [] + stdin_, stdout_, stderr_ = None, None, None + while i < max_tries: + try: + if verbose: + logger.debug(command) + stdin_, stdout_, stderr_ = remote.exec_command(command, timeout=timeout) + # Block until the client finishes + stdout_.channel.recv_exit_status() + error_lines = stderr_.readlines() + lines = stdout_.readlines() + break + # We would like to catch paramiko.SSHException here, but because of issues with importing paramiko + # at the top of the file in the main namespace, we catch Exception instead as a broader exception. + except Exception: + logger.error("Failed to execute \"%s\" on remote host. Re-creating SSH tunnel." % command) + # Note that at this point, our remote object still has a valid channel as far as paramiko is + # concerned, but the channel is no longer responding. If we tried to do remote.close() here, + # it would hang our process. Instead, we'll set up a new remote channel, and let Python's garbage + # collection handle destroying the original remote object for us. + try: + remote = self.setup_remote() + except Exception: + logger.error("Failed to re-connect remote - trying again.") + i += 1 + + if verbose: + for line in error_lines: + logger.debug("ERROR: %s", line.strip()) + # Close the channels + if stdin_: + stdin_.close() + if stdout_: + stdout_.close() + if stderr_: + stderr_.close() + return lines, error_lines + + def get_log(self, remote, worker, log_name, logger): + """ + Retrieves a log from the remote server and writes it to disk. + + Args: + remote: A Paramiko SSH channel to execute over + worker (dict): Dictionary describing external client worker + log_name (str): Log name to retrieve + logger (:obj:`logging.Logger`): A logger to log with + """ + # Get the client.log + log_path = os.path.join(self.output_directory, "logs", log_name) + command = "cat %s" % os.path.join(worker["geneva_path"], log_path) + client_log, error_lines = self.remote_exec_cmd(remote, command, logger, verbose=False) + # If something goes wrong, we don't necessarily want to dump the entire client_log to the screen + # a second time, so just disable verbosity and display the stderr. + for line in error_lines: + logger.error(line.strip()) + + # Write the client log out to disk + with open(log_path, "w") as fd: + for line in client_log: + fd.write(line) + + def run_local_client(self, args, environment, logger): + """ + Runs client locally. Does not return fitness. + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + """ + # Launch the plugin client + command = [sys.executable, "plugins/plugin_client.py", "--plugin", self.client_cls.name] + base_cmd = actions.utils.build_command(args) + command += base_cmd + # Replace strings of empty strings "''" with empty strings "", as subprocess will handle this correctly + command = [x if x != "''" else "" for x in command] + logger.debug(" ".join(command)) + self.exec_cmd(command) + + def exec_cmd(self, command, timeout=60): + """ + Runs a subprocess command at the correct log level. + + Args: + command (list): Command to execute. + timeout (int, optional): Timeout for execution + """ + self.logger.debug(" ".join(command)) + try: + if actions.utils.get_console_log_level() == "debug": + subprocess.check_call(command, timeout=60) + else: + subprocess.check_call(command, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=60) + except subprocess.CalledProcessError as exc: + # Code 137 is for SIGKILL, which is how docker containers are shutdown by the evaluator. + # Ignore these exceptions, raise all others + if exc.returncode != 137: + raise + + def exec_cmd_output(self, command, timeout=60): + """ + Runs a subprocess command at the correct log level. This is a separate method from the above exec_cmd, + since that is used to stream output to the screen (so check_output is not appropriate). + + Args: + command (list): Command to execute. + timeout (int, optional): Timeout for execution + + Returns: + str: Output of command + """ + self.logger.debug(" ".join(command)) + output = "" + try: + output = subprocess.check_output(command, timeout=60, stderr=subprocess.PIPE).decode('utf-8', 'ignore') + if actions.utils.get_console_log_level() == "debug": + self.logger.debug(output) + except subprocess.CalledProcessError as exc: + # Code 137 is for SIGKILL, which is how docker containers are shutdown by the evaluator. + # Ignore these exceptions, raise all others + if exc.returncode != 137: + raise + return output + + def start_server(self, args, environment, logger): + """ + Launches the server. + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + + Return: + float: fitness of individual (if one is provided) + """ + if environment.get("docker"): + logger.debug("Evaluator: running server inside docker") + return self.run_docker_server(args, environment, logger) + else: + logger.debug("Evaluator: running server") + return self.run_local_server(args, environment, logger) + + def run_docker_server(self, args, environment, logger): + """ + Runs server and censor in their respective docker containers. + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + """ + command = ["docker", "exec", "--privileged", environment["server"]["container"].name, "python", "code/plugins/plugin_server.py", "--test-type", self.server_cls.name] + base_cmd = actions.utils.build_command(args) + command += base_cmd + # Replace strings of empty strings "''" with empty strings "", as subprocess will handle this correctly + command = [x if x != "''" else "" for x in command] + + port = args.get("port") + queue_num = random.randint(1, 1000) + environment["port"] = port + environment["queue_num"] = queue_num + server_thread = threading.Thread(target=self.exec_cmd, args=(command, )) + censor_thread = threading.Thread(target=self.start_censor, args=(environment, environment["id"])) + censor_thread.start() + server_thread.start() + max_wait = 30 + count = 0 + flag_file = os.path.join(args["output_directory"], "flags", "%s.server_ready" % args["environment_id"]) + + while count < max_wait: + if os.path.exists(flag_file): + break + if count % 15 == 0: + logger.debug("Evaluator waiting for confirmation of server startup") + count += 1 + time.sleep(0.5) + else: + logger.warn("Evaluator: Server did not startup within window") + return + logger.debug("Evaluator: Server ready.") + + def stop_server(self, environment, server): + """ + Stops server. + + Args: + environment (dict): Environment dictionary + server (:obj:`plugins.plugin_server.ServerPlugin`): A plugin server to stop + """ + # If the server is running inside a docker container, we don't have access to it directly + # to shut it down. Instead, write a shutdown flag to instruct it to shut down. + self.logger.debug("Evaluator shutting down server.") + if environment.get("docker"): + flag_file = os.path.join(self.args["output_directory"], "flags", "%s.server_shutdown" % self.server_args["environment_id"]) + # Touch shutdown file to instruct the server to shutdown + open(flag_file, 'a').close() + self.stop_censor(environment) + else: + # Shut down the server + server.stop() + # Shut down the server's logger, now that we are done with it + actions.utils.close_logger(environment["server_logger"]) + + def run_local_server(self, args, environment, logger): + """ + Runs local server. + + Args: + args (dict): Dictionary of arguments + environment (dict): Dictionary describing environment configuration for this evaluation + logger (:obj:`logging.Logger`): A logger to log with + """ + server = self.server_cls(args) + logger.debug("Starting local server with args: %s" % str(args)) + server_logger = actions.utils.get_logger(PROJECT_ROOT, args["output_directory"], "server", "server", environment["id"], log_level=actions.utils.get_console_log_level()) + environment["server_logger"] = server_logger + args.update({"test_type": self.server_cls.name}) + if not args.get("server_side"): + args.update({"no_engine" : True}) + server.start(args, server_logger) + return server + + def canary_phase(self, canary): + """ + Learning phase runs the client against the censor to collect packets. + + Args: + canary (:obj:`actions.strategy.Strategy`): A (usually empty) strategy object to evaluate + + Returns: + str: canary id used ("canary") + """ + if not self.run_canary_phase: + return None + + self.logger.info("Starting collection phase") + environment = {} + canary.environment_id = "canary" + if self.use_docker: + try: + environment = self.create_test_environment("canary") + except (docker.errors.APIError, requests.exceptions.ConnectionError, urllib3.exceptions.ProtocolError): + self.logger.error("Failed to create evaluator environment - is docker running?") + return None + + self.worker([canary], canary.environment_id, environment) + + self.logger.info("Collected packets under %s" % canary) + return "canary" + + def get_ip(self): + """ + Gets IP of evaluator computer. + + Returns: + str: Public IP provided + """ + if self.public_ip: + return self.public_ip + return None + + def create_test_environment(self, worker_id): + """ + Creates a test environment in docker. + + Args: + worker_id (int): Worker ID of this worker + + Returns: + dict: Environment dictionary to use + """ + self.logger.debug("Initializing docker environment.") + + # We can't have an environment with an intenral test server and no censor + # with the current set up. To be addressed later to allow for no censor testing + assert not (not self.censor and not self.external_server), "Can't create internal server w/o censor" + assert not (self.censor and self.external_server), "Can't create a censor without an internal training server" + + # Create a dict to hold the environment we're about to create + environment = {} + + # Create the client container + environment["client"] = self.initialize_base_container("client_%s" % worker_id) + environment["client"]["ip"] = self.parse_ip(environment["client"]["container"], "eth0") + + # If a training censor is requested, create a censor container + if self.censor: + environment["censor"] = self.initialize_base_container("censor_%s" % worker_id) + environment["server"] = self.initialize_base_container("server_%s" % worker_id) + # Set up the routing + environment["server"]["ip"] = self.parse_ip(environment["server"]["container"], "eth0") + environment["censor"]["ip"] = self.parse_ip(environment["censor"]["container"], "eth0") + self._add_route(environment["server"]["container"], environment["censor"]["ip"]) + self._add_route(environment["client"]["container"], environment["censor"]["ip"]) + + # Calculate the network base ("172.17.0.0") + network_base = ".".join(environment["server"]["ip"].split(".")[:2]) + ".0.0" + + # Delete all other routes for the server and client to force communication through the censor + environment["server"]["container"].exec_run(["route", "del", "-net", network_base, "gw", "0.0.0.0", "netmask", "255.255.0.0", "dev", "eth0"], privileged=True) + environment["client"]["container"].exec_run(["route", "del", "-net", network_base, "gw", "0.0.0.0", "netmask", "255.255.0.0", "dev", "eth0"], privileged=True) + + # Set up NAT on the censor + environment["censor"]["container"].exec_run(["iptables", "-t", "nat", "-A", "POSTROUTING", "-j", "MASQUERADE"], privileged=True) + + self.environments.append(environment) + # Flag that this environment is a docker environment + environment["docker"] = True + # Return the configured environment for use + return environment + + def _add_route(self, container, via): + """ + Helper method to take down an interface on a container + + Args: + container: Docker container object to execute within + via (str): IP address to route through + """ + exit_code, _output = container.exec_run(["ip", "route", "del", "default"], privileged=True) + exit_code, _output = container.exec_run(["ip", "route", "add", "default", "via", via], privileged=True) + return exit_code + + def parse_ip(self, container, iface): + """ + Helper method to parse an IP address from ifconfig. + + Args: + container: Docker container object to execute within + iface (str): Interface to parse from + + Returns: + str: IP address + """ + _exit_code, output = container.exec_run(["ifconfig", iface], privileged=True) + ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', output.decode("utf-8"))[0] + return ip + + def setup_remote(self): + """ + Opens an SSH tunnel to the remote client worker. + """ + # Import paramiko here instead of at the top of the file. This is done intentionally. When + # paramiko is imported, pynacl is loaded, which polls /dev/random for entropy to setup crypto + # keys. However, if the evaluator is run on a relatively blank VM (or AWS instance) with little + # network traffic before it starts (as will often be the case), there may be insufficient entropy + # available in the system. This will cause pynacl to block on entropy, and since the only thing + # running on the system is now blocking, it is liable to block indefinitely. Instead, the import + # is performed here so that the system interaction of running the evaluator this far collects + # enough entropy to not block paramiko. The pynacl team is aware of this issue: see issue #503 + # (https://github.com/pyca/pynacl/issues/503) and #327 (https://github.com/pyca/pynacl/issues/327) + import paramiko + paramiko_logger = paramiko.util.logging.getLogger() + paramiko_logger.setLevel(logging.WARN) + worker = actions.utils.get_worker(self.external_client, self.logger) + if self.use_docker: + worker["ip"] = "0.0.0.0" + self.logger.debug("Connecting to worker %s@%s" % (worker["username"], worker["ip"])) + remote = paramiko.SSHClient() + remote.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + max_tries = 5 + i = 0 + while i < max_tries: + try: + if "keyfile" in worker: + k = paramiko.RSAKey.from_private_key_file(worker["keyfile"]) + remote.connect(worker["ip"], username=worker["username"], pkey=k, port=worker["port"], timeout=60) + else: + remote.connect(worker["ip"], username=worker["username"], password=worker["password"], port=worker["port"], timeout=60) + break + except socket.timeout: + self.logger.error("Could not connect to worker %s" % worker["ip"]) + i += 1 + return remote + + def worker(self, ind_list, worker_id, environment): + """ + Perform the actual fitness evaluation as a multithreaded worker. The + worker pops off an individual from the list and evaluates it. + + Args: + ind_list (list): List of strategy objects to evaluate + worker_id (int): ID of this worker + environment (dict): Environment dictionary + """ + environment["remote"] = None + + if self.external_client: + environment["remote"] = self.setup_remote() + environment["worker"] = actions.utils.get_worker(self.external_client, self.logger) + + for ind in ind_list: + if self.stop: + break + + # Run a test + eid, fitness = self.run_test(environment, ind) + + if not fitness: + fitness = -1000 + + # Dump logs if requested + if fitness < 0 and self.args.get("log_on_fail"): + self.dump_logs(eid) + elif fitness > 0 and self.args.get("log_on_success"): + self.dump_logs(eid) + + # Clean up the test environment + self.shutdown_environment(environment) + + def assign_ids(self, ind_list): + """ + Assigns random environment ids to each individual to be evaluated. + + Args: + ind_list (list): List of individuals to assign random IDs to + """ + for ind in ind_list: + ind.environment_id = actions.utils.get_id() + ind.fitness = None + + def dump_logs(self, environment_id): + """ + Dumps client, engine, server, and censor logs, to be called on test failure + at ERROR level. + + Args: + environment_id (str): Environment ID of a strategy to dump + """ + log_files = ["client.log", "engine.log", "censor.log", "server.log"] + for log_file in log_files: + log = "" + log_path = os.path.join(BASEPATH, + self.output_directory, + "logs", + "%s.%s" % (environment_id, log_file)) + try: + if not os.path.exists(log_path): + continue + with open(log_path, "rb") as logfd: + log = logfd.read().decode('utf-8') + except Exception: + self.logger.exception("Failed to open log file") + continue + self.logger.error("%s: %s", log_file, log) + + def terminate_docker(self): + """ + Terminates any hanging running containers. + """ + if not self.use_docker: + return + # First, stop all the docker containers that match the given names + # If a previous run was cut off in between container creation and startup, + # we must also remove the container ('docker rm ') + for operation in ["stop", "rm"]: + try: + output = subprocess.check_output(['docker', 'ps', '--format', "'{{.Names}}'"]).decode('utf-8') + except subprocess.CalledProcessError: + self.logger.error("Failed to list container names -- is docker running?") + return + if output.strip(): + self.logger.debug("Cleaning up docker (%s)" % operation) + for name in output.splitlines(): + if any(key in name for key in ["client", "censor", "server"]): + try: + subprocess.check_output(['docker', operation, name]) + except subprocess.CalledProcessError: + pass + + def initialize_base_container(self, name): + """ + Builds a base container with a given name and connects it to a given network. + Also retrieves lower level settings and the IP address of the container. + + Args: + name (str): Name of this docker container + + Returns: + dict: Dictionary containing docker container object and relevant information + """ + try: + container = {} + container["name"] = name + # Note that this is _not_ safe to do in a multiprocessed context - must be run single threaded. + container["container"] = self.docker_client.containers.run('base', detach=True, privileged=True, volumes={os.path.abspath(os.getcwd()): {"bind" : "/code", "mode" : "rw"}}, tty=True, remove=True, name=name) + container["settings"] = self.apiclient.inspect_container(name) + except docker.errors.NotFound: + self.logger.error("Could not run container \"base\". Is docker not running, or does the base container not exist?") + return None + + return container + + def get_pid(self, container): + """ + Returns PID of first actively running python process. + + Args: + container: Docker container object to query + + Returns: + int: PID of Python process + """ + pid = None + try: + output = subprocess.check_output(["docker", "exec", container.name, "ps", "aux"], stderr=subprocess.PIPE).decode('utf-8') + except subprocess.CalledProcessError: + return None + for line in output.splitlines(): + if "root" not in line or "python" not in line: + continue + parts = line.split() + # Try to parse out the pid to confirm we found it + try: + pid = int(parts[1]) + break + except ValueError: + raise + return pid + + def stop_censor(self, environment): + """ + Send SIGKILL to all remaining python processes in the censor container. + This is done intentionally over a SIGINT or a graceful shutdown mecahnism - due to + dynamics with signal handling in nfqueue callbacks (threads), SIGINTs can be ignored + and graceful shutdown mechanisms may not be picked up (or be fast enough). + + The output this method parses is below: + + .. code-block:: bash + + # ps aux + USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND + root 1 0.1 0.0 21944 3376 pts/0 Ss+ 13:30 0:00 /bin/bash + root 14 24.0 0.4 200376 38564 ? Ss 13:30 0:00 python code/censor_driver.py censor2 jgsko1rf trials/2018-10-30_06:30:48 60181 + root 32 0.0 0.0 19188 2400 ? Rs 13:30 0:00 ps aux + + Args: + environment (dict): Environment dictionary + """ + port = environment["port"] + queue_num = environment["queue_num"] + if environment: + pid = self.get_pid(environment["censor"]["container"]) + while pid: + #self.logger.info("%s killing process %s in %s" % (environment["id"], str(pid), environment["censor"]["container"].name)) + try: + subprocess.check_call(["docker", "exec", "--privileged", environment["censor"]["container"].name, "kill", "-9", str(pid)]) + except subprocess.CalledProcessError: + pass + pid = self.get_pid(environment["censor"]["container"]) + time.sleep(0.25) + + try: + subprocess.check_call(["docker", "exec", "--privileged", environment["censor"]["container"].name, "iptables", "-D", "FORWARD", "-j", "NFQUEUE", "-p", "tcp", "--sport", str(port), "--queue-num", str(queue_num)]) + except subprocess.CalledProcessError: + pass + try: + subprocess.check_call(["docker", "exec", "--privileged",environment["censor"]["container"].name, "iptables", "-D", "FORWARD", "-j", "NFQUEUE", "-p", "tcp", "--dport", str(port), "--queue-num", str(queue_num)]) + except subprocess.CalledProcessError: + pass + + def start_censor(self, environment, environment_id): + """ + Starts the censor in the server environment container. + + Args: + environment (dict): Environment dictionary + environment_id (str): Environment ID of the censor to stop + """ + assert self.use_docker, "Cannot start censor without enabling docker" + port = environment["port"] + queue_num = environment["queue_num"] + try: + self.logger.debug(" Starting censor %s with driver" % self.censor) + command = ["docker", "exec", "--privileged", environment["censor"]["container"].name, + "python", "code/censors/censor_driver.py", + "--censor", self.censor, + "--environment-id", environment_id, + "--output-directory", self.output_directory, + "--port", str(port), + "--log", "debug", + "--forbidden", self.args.get("bad_word", "ultrasurf"), + "--queue", str(queue_num)] + self.exec_cmd(command) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + # Docker containers were killed out from under us - likely means + # user forced a shutdown. Bail gracefully. + return False + except Exception: + self.logger.exception("Failed out of start_censor") + finally: + self.logger.debug("Dockerized censor thread exiting") + + def read_fitness(self, ind): + """ + Looks for this individual's fitness file on disk, opens it, and stores the fitness in the given individual. + + Args: + ind (:obj:`actions.strategy.Strategy`): Individual to read fitness for + """ + fitness_path = os.path.join(BASEPATH, self.output_directory, actions.utils.FLAGFOLDER, ind.environment_id + ".fitness") + try: + if os.path.exists(fitness_path): + with open(fitness_path, "r") as fd: + ind.fitness = float(fd.read()) + elif not ind.fitness: + self.logger.warning("Could not find fitness file for %s" % fitness_path) + ind.fitness = -1000 + except: + self.logger.exception("[%s] Failed to read fitness file" % ind.environment_id) + ind.fitness = -1000 + + def shutdown_container(self, container): + """ + Tries to shutdown a given container and eats a NotFound exception if the container + has already exited. + + Args: + container: docker container object to call stop() on + """ + try: + container.stop() + except docker.errors.NotFound: + pass + + def shutdown_environment(self, environment): + """ + Shuts down the evaluation environment. + If Docker, shuts down server and client container. + If a remote SSH connection, the connection is shut down. + """ + if environment.get("docker"): + self.shutdown_container(environment["client"]["container"]) + if self.censor: + self.shutdown_container(environment["censor"]["container"]) + self.shutdown_container(environment["server"]["container"]) + elif environment.get("remote"): + environment["remote"].close() + + def shutdown(self): + """ + Shuts down all active environments + """ + self.terminate_docker() + + +def collect_plugin(test_plugin, plugin_type, command, full_args, plugin_args): + """ + Import a given plugin + + Args: + test_plugin (str): Plugin name to import ("http") + plugin_type (str): Component of plugin to import ("client") + command (list): sys.argv or list of arguments + full_args (dict): Parsed full list of arguments already maintained by the parent plugin + plugin_args (dict): Dictionary of args specific to this plugin component + + Returns: + Imported plugin class for instantiation later + """ + cls = None + try: + _, cls = actions.utils.import_plugin(test_plugin, plugin_type) + parsed_args = cls.get_args(command) + # Only override the args if the plugin successfully parsed something; this allows + # defaults from the evaluator or plugin to propagate. + parsed_args = {k:v for k,v in parsed_args.items() if v or (not v and k not in full_args) } + full_args.update(parsed_args) + plugin_args.update(parsed_args) + except ImportError as exc: + pass + return cls + + +def get_random_open_port(): + """ + Selects a random ephemeral port that is open. + + Returns: + int: Open port + """ + while True: + port = random.randint(1024, 65000) + # Bind TCP socket + try: + with socket.socket() as sock: + # If we can bind, nothing is listening + sock.bind(('', port)) + break + except OSError: + continue + return port + + +def get_arg_parser(single_use=False): + """ + Sets up argparse. This is done separately to enable collection of help messages. + + Args: + single_use (bool, optional): whether this evaluator will only be used for one strategy, used to configure sane defaults + """ + # Disable prefix matching to avoid prefix collisions for unseen plugin arguments + parser = argparse.ArgumentParser(description='Evaluate a given strategy a given number of times.', allow_abbrev=False, prog="evaluator.py") + # Type of evaluation + parser.add_argument('--test-type', action='store', choices=actions.utils.get_plugins(), default="http", help="plugin to launch") + parser.add_argument('--strategy', action='store', default="", required=single_use, help='strategy to evaluate') + + logging_group = parser.add_argument_group('control aspects of evaluator logging and storage') + logging_group.add_argument('--log', action='store', choices=("debug", "info", "warning", "critical", "error"), help="Sets the log level") + logging_group.add_argument('--output-directory', action='store', help="where to output results") + logging_group.add_argument('--log-on-fail', action='store_true', help="dump the logs associated with each individual on strategy failure") + logging_group.add_argument('--log-on-success', action='store_true', help="dump the logs associated with each individual on strategy success") + + external_group = parser.add_argument_group('control aspects of external resource usage') + external_group.add_argument('--external-server', action='store_true', help="use an external server for testing.") + external_group.add_argument('--external-client', action='store', help="use the given external client for testing.") + + networking_group = parser.add_argument_group('control aspects of evaluator networking configuration') + networking_group.add_argument('--server-side', action="store_true", help="run the Geneva engine on the server side, not the client") + networking_group.add_argument('--public-ip', action='store', help="public facing IP for this computer for server-side evaluation.") + networking_group.add_argument('--routing-ip', action='store', help="locally facing IP for this computer, used for NAT") + networking_group.add_argument('--sender-ip', action='store', help="IP address of sending machine, used for NAT") + networking_group.add_argument('--forward-ip', action='store', help="IP address to forward traffic to") + networking_group.add_argument('--act-as-middlebox', action='store_true', help="enables NAT mode. Requires --routing-ip, --sender-ip, and --forward-ip") + networking_group.add_argument('--port', action='store', type=int, default=get_random_open_port(), help='default port to use') + + docker_group = parser.add_argument_group('control aspects of docker-specific options') + docker_group.add_argument('--censor', action='store', help='censor to test against.', choices=censors.censor_driver.get_censors()) + docker_group.add_argument('--workers', action='store', default=1, type=int, help='controls the number of docker containers the evaluator will use.') + docker_group.add_argument('--bad-word', action='store', help="forbidden word to test with", default="ultrasurf") + + evaluation_group = parser.add_argument_group('control aspects of evaluation') + evaluation_group.add_argument('--runs', type=int, default=1, action='store', help="number of times each individual should be run per evaluation") + evaluation_group.add_argument("--fitness-by", action='store', choices=('min', 'avg', 'max'), default='avg', help="if each individual is run multiple times, control how fitness is assigned.") + evaluation_group.add_argument('--no-skip-empty', action='store_true', help="evaluate empty strategies (default: False).") + + return parser + + +def get_args(cmd, single_use=False): + """ + Creates an argparser and collects arguments. + + Args: + single_use (bool, optional): whether this evaluator will only be used for one strategy, used to configure sane defaults + + Returns: + dict: parsed args + """ + parser = get_arg_parser(single_use=single_use) + + args, _ = parser.parse_known_args(cmd) + return vars(args) diff --git a/evolve.py b/evolve.py new file mode 100644 index 0000000..16b8002 --- /dev/null +++ b/evolve.py @@ -0,0 +1,834 @@ +""" +Main evolution driver for Geneva (GENetic EVAsion). This file performs the genetic algorithm, +and relies on the evaluator (evaluator.py) to provide fitness evaluations of each individual. +""" + +import argparse +import copy +import logging +import operator +import os +import random +import subprocess as sp +import sys + +import actions.strategy +import actions.tree +import actions.trigger +import evaluator + +# Grab the terminal size for printing +try: + _, COLUMNS = sp.check_output(['stty', 'size']).decode().split() +# If pytest has capturing enabled or this is run without a tty, catch the exception +except sp.CalledProcessError: + _, COLUMNS = 0, 0 + + +def setup_logger(log_level): + """ + Sets up the logger. This will log at the specified level to "ga.log" and at debug level to "ga_debug.log". + Logs are stored in the trials/ directory under a run-specific folder. + Example: trials/2020-01-01_01:00:00/logs/ga.log + + Args: + log_level (str): Log level to use in setting up the logger ("debug") + """ + level = log_level.upper() + assert level in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], "Unknown log level %s" % level + actions.utils.CONSOLE_LOG_LEVEL = level.lower() + + # Setup needed folders + ga_log_dir = actions.utils.setup_dirs(actions.utils.RUN_DIRECTORY) + + ga_log = os.path.join(ga_log_dir, "ga.log") + ga_debug_log = os.path.join(ga_log_dir, "ga_debug.log") + + # Configure logging globally + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s:%(message)s', datefmt="%Y-%m-%d %H:%M:%S") + logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', datefmt="%Y-%m-%d %H:%M:%S") + + # Set up the root logger + logger = logging.getLogger("ga_%s" % actions.utils.RUN_DIRECTORY) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + setattr(logger, "ga_log_dir", ga_log_dir) + + # If this logger's handlers have already been set up, don't add them again + if logger.handlers: + return logger + + # Set log level of console + console = logging.StreamHandler() + console.setLevel(level) + console.setFormatter(formatter) + logger.addHandler(console) + + # Add a DEBUG file handler to send all the debug output to a file + debug_file_handler = logging.FileHandler(ga_debug_log) + debug_file_handler.setFormatter(formatter) + debug_file_handler.setLevel(logging.DEBUG) + logger.addHandler(debug_file_handler) + + # Add a file handler to send all the output to a file + file_handler = logging.FileHandler(ga_log) + file_handler.setFormatter(formatter) + file_handler.setLevel(level) + logger.addHandler(file_handler) + return logger + + +def collect_plugin_args(cmd, plugin, plugin_type, message=None): + """ + Collects and prints arguments for a given plugin. + + Args: + cmd (list): sys.argv or a list of args to parse + plugin (str): Name of plugin to import ("http") + plugin_type (str): Component of plugin to import ("client") + message (str): message to override for printing + """ + if not message: + message = plugin_type + try: + _, cls = actions.utils.import_plugin(plugin, plugin_type) + print("\n\n") + print("=" * int(COLUMNS)) + print("Options for --test-type %s %s" % (plugin, message)) + cls.get_args(cmd) + # Catch SystemExit here, as this is what argparse raises when --help is passed + except (SystemExit, Exception): + pass + + +def get_args(cmd): + """ + Sets up argparse and collects arguments. + + Args: + cmd (list): sys.argv or a list of args to parse + + Returns: + namespace: Parsed arguments + """ + parser = argparse.ArgumentParser(description='Genetic algorithm for evolving censorship evasion.\n\nevolve.py uses a pass-through argument system to pass the command line arguments through different files in the system, including the evaluator (evaluator.py) and a given plugin (plugins/). --help will collect all these arguments.', add_help=False, prog="evolve.py") + + parser.add_argument('--test-type', action='store', choices=actions.utils.get_plugins(), default="http", help="plugin to launch") + + # Add help message separately so we can collect the help messages of all of the other parsers + parser.add_argument('-h', '--help', action='store_true', default=False, help='print this help message and exit') + + # Control aspects of individuals + ind_group = parser.add_argument_group('control aspects of individual strategies') + ind_group.add_argument('--in-trees', action='store', type=int, default=0, help='starting # of input-direction action trees per strategy. Disables inbound forest if set to 0') + ind_group.add_argument('--out-trees', action='store', type=int, default=1, help='starting # of output-direction action trees per strategy') + ind_group.add_argument('--in-actions', action='store', type=int, default=2, help='starting # of input-direction actions per action tree') + ind_group.add_argument('--out-actions', action='store', type=int, default=2, help='starting # of output-direction actions per action tree') + ind_group.add_argument('--fix-trigger', action='store', help='fix all triggers for this evolution to a given trigger') + + # Options to control the population pool + pop_group = parser.add_argument_group('control aspects of the population pool') + pop_group.add_argument('--load-from', action='store', help="Load population from a generation file") + pop_group.add_argument('--seed', action='store', help='seed strategy to initialize the population to.') + + # Options to evaluate and exit, skip evaluation, and to specify the type of test + evaluation_group = parser.add_argument_group('control aspects of strategy evaluation') + evaluation_group.add_argument('--eval-only', action='store', default=None, help='only evaluate fitness for a given strategy or file of strategies') + evaluation_group.add_argument('--no-eval', action='store_true', help="Disable evaluator for debugging") + evaluation_group.add_argument('--runs', action='store', type=int, default=1, help='number of times each strategy should be run for one evaluation (default 1, fitness is averaged).') + + # Hyperparameters for genetic algorithm + genetic_group = parser.add_argument_group('control aspects of the genetic algorithm') + genetic_group.add_argument('--elite-clones', action='store', type=int, default=3, help='number copies of the highest performing individual that should be propagated to the next generation.') + genetic_group.add_argument('--mutation-pb', action='store', type=float, default=0.99, help='mutation probability') + genetic_group.add_argument('--crossover-pb', action='store', type=float, default=0.4, help='crossover probability') + genetic_group.add_argument('--allowed-retries', action='store', type=int, default=20, help='maximum number of times GA will generate any given individual') + genetic_group.add_argument('--generations', type=int, action='store', default=50, help="number of generations to run for.") + genetic_group.add_argument('--population', type=int, action='store', default=250, help="size of population.") + genetic_group.add_argument('--no-reject-empty', action='store_true', default=False, help="disable mutation rejection of empty strategies") + genetic_group.add_argument('--no-canary', action='store_true', help="disable canary phase") + + # Limit access to certain protocols, fields, actions, or types of individuals + filter_group = parser.add_argument_group('limit access to certain protocols, fields, actions, or types of individuals') + filter_group.add_argument('--protos', action="store", default="TCP", help="allow the GA to scope only to these protocols") + filter_group.add_argument('--fields', action='store', default="", help='restrict the GA to only seeing given fields') + filter_group.add_argument('--disable-fields', action='store', default="", help='restrict the GA to never using given fields') + filter_group.add_argument('--no-gas', action="store_true", help="disables trigger gas") + filter_group.add_argument('--disable-action', action='store', default="sleep,trace", help='disables specific actions') + + # Logging + logging_group = parser.add_argument_group('control logging') + logging_group.add_argument('--log', action='store', default="info", choices=("debug", "info", "warning", "critical", "error"), help="Sets the log level") + logging_group.add_argument('--no-print-hall', action='store_true', help="does not print hall of fame at the end") + logging_group.add_argument('--graph-trees', action='store_true', default=False, help='graph trees in addition to outputting to screen') + + # Misc + usage_group = parser.add_argument_group('misc usage') + usage_group.add_argument('--no-lock-file', default=(os.name == "posix"), action='store_true', help="does not use /lock_file.txt") + usage_group.add_argument('--force-cleanup', action='store_true', default=False, help='cleans up all docker containers and networks after evolution') + + if not cmd: + parser.error("No arguments specified") + + args, _ = parser.parse_known_args(cmd) + + epilog = "See the README.md for usage." + # Override the help message to collect the pass through args + if args.help: + parser.print_help() + print(epilog) + print("=" * int(COLUMNS)) + print("\nevolve.py uses a pass-through argument system to evaluator.py and other parts of Geneva. These arguments are below.\n\n") + evaluator.get_arg_parser(cmd).print_help() + if args.test_type: + collect_plugin_args(cmd, args.test_type, "plugin", message="parent plugin") + collect_plugin_args(cmd, args.test_type, "client") + collect_plugin_args(cmd, args.test_type, "server") + raise SystemExit + return args + + +def fitness_function(logger, population, ga_evaluator): + """ + Calls the evaluator to evaluate a given population of strategies. + Sets the .fitness attribute of each individual. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + population (list): List of individuals to evaluate + ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator object to evaluate with + + Returns: + list: Population post-evaluation + """ + if ga_evaluator: + return ga_evaluator.evaluate(population) + + for ind in population: + ind.fitness = 0 + logger.info("[%s] Fitness %d: %s", -1, ind.fitness, str(ind)) + + return population + + +def sel_random(individuals, k): + """ + Implementation credit to DEAP: https://github.com/DEAP/deap + Select *k* individuals at random from the input *individuals* with + replacement. The list returned contains references to the input + *individuals*. + + Args: + individuals (list): A list of individuals to select from. + k (int): The number of individuals to select. + + Returns: + list: A list of selected individuals. + """ + return [random.choice(individuals) for _ in range(k)] + + +def selection_tournament(individuals, k, tournsize, fit_attr="fitness"): + """ + Implementation credit to DEAP: https://github.com/DEAP/deap + Select the best individual among *tournsize* randomly chosen + individuals, *k* times. The list returned contains + references to the input *individuals*. + + Args: + individuals (list): A list of individuals to select from. + k (int): The number of individuals to select. + tournsize (int): The number of individuals participating in each tournament. + fit_attr: The attribute of individuals to use as selection criterion (defaults to "fitness") + + Returns: + list: A list of selected individuals. + """ + chosen = [] + for _ in range(k): + aspirants = sel_random(individuals, tournsize) + chosen.append(copy.deepcopy(max(aspirants, key=operator.attrgetter(fit_attr)))) + return chosen + + +def get_unique_population_size(population): + """ + Computes number of unique individuals in a population. + + Args: + population (list): Population list + """ + uniques = {} + for ind in population: + uniques[str(ind)] = True + return len(list(uniques.keys())) + + +def add_to_hof(hof, population): + """ + Iterates over the current population and updates the hall of fame. + The hall of fame is a dictionary that tracks the fitness of every + run of every strategy ever. + + Args: + hof (dict): Current hall of fame + population (list): Population list + + Returns: + dict: Updated hall of fame + """ + for ind in population: + if str(ind) not in hof: + hof[str(ind)] = [] + hof[str(ind)].append(ind.fitness) + + return hof + + +def generate_strategy(logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, environment_id=None, disabled=None): + """ + Generates a strategy individual. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + num_in_trees (int): Number of trees to initialize in the inbound forest + num_out_trees (int): Number of trees to initialize in the outbound forest + num_in_actions (int): Number of actions to initialize in the each inbound tree + num_out_actions (int): Number of actions to initialize in the each outbound tree + environment_id (str, optional): Environment ID to assign to the new individual + disabled (str, optional): List of actions that should not be considered in building a new strategy + + Returns: + :obj:`actions.strategy.Strategy`: A new strategy object + """ + try: + strat = actions.strategy.Strategy([], [], environment_id=environment_id) + strat.initialize(logger, num_in_trees, num_out_trees, num_in_actions, num_out_actions, seed, disabled=disabled) + except Exception: + logger.exception("Failure to generate strategy") + raise + + return strat + + +def mutation_crossover(logger, population, hall, options): + """ + Apply crossover and mutation on the offspring. + + Hall is a copy of the hall of fame, used to accept or reject mutations. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + population (list): Population of individuals + hall (dict): Current hall of fame + options (dict): Options to override settings. Accepted keys are: + "crossover_pb" (float): probability of crossover + "mutation_pb" (float): probability of mutation + "allowed_retries" (int): number of times a strategy is allowed to exist in the hall of fame. + "no_reject_empty" (bool): whether or not empty strategies should be rejected + + Returns: + list: New population after mutation + """ + cxpb = options.get("crossover_pb", 0.5) + mutpb = options.get("mutation_pb", 0.5) + + offspring = copy.deepcopy(population) + for i in range(1, len(offspring), 2): + if random.random() < cxpb: + ind = offspring[i - 1] + actions.strategy.mate(ind, offspring[i], indpb=0.5) + offspring[i - 1].fitness, offspring[i].fitness = -1000, -1000 + + for i in range(len(offspring)): + if random.random() < mutpb: + + mutation_accepted = False + while not mutation_accepted: + test_subject = copy.deepcopy(offspring[i]) + mutate_individual(logger, test_subject) + + # Pull out some metadata about this proposed mutation + fitness_history = hall.get(str(test_subject), []) + + # If we've seen this strategy 10 times before and it has always failed, + # or if we have seen it 20 times already, or if it is an empty strategy, + # reject this mutation and get another + if len(fitness_history) >= 10 and all(fitness < 0 for fitness in fitness_history) or \ + len(fitness_history) >= options.get("allowed_retries", 20) or \ + (len(test_subject) == 0 and not options.get("no_reject_empty")): + mutation_accepted = False + else: + mutation_accepted = True + + offspring[i] = test_subject + offspring[i].fitness = -1000 + + return offspring + + +def mutate_individual(logger, ind): + """ + Simply calls the mutate function of the given individual. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + ind (:obj:`actions.strategy.Strategy`): A strategy individual to mutate + + Returns: + :obj:`actions.strategy.Strategy`: Mutated individual + """ + return ind.mutate(logger) + + +def run_collection_phase(logger, ga_evaluator): + """Individual mutation works best when it has seen real packets to base + action and trigger values off of, instead of blindly fuzzing packets. + Usually, the 0th generation is useless because it hasn't seen any real + packets yet, and it bases everything off fuzzed data. To combat this, a + canary phase is done instead. + + In the canary phase, a single dummy individual is evaluated to capture + packets. Once the packets are captured, they are associated with all of the + initial population pool, so all of the individuals have some packets to base + their data off of. + + Since this phase by necessity requires the evaluator, this is only run if + --no-eval is not specified. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator object to evaluate with + + Returns: + str: ID of the test 'canary' strategy evaluated to do initial collection + """ + canary = generate_strategy(logger, 0, 0, 0, 0, None, disabled=[]) + canary_id = ga_evaluator.canary_phase(canary) + if not canary_id: + return [] + return canary_id + + +def write_generation(filename, population): + """ + Writes the population pool for a specific generation. + + Args: + filename (str): Name of file to write the generation out to + population (list): List of individuals to write out + """ + # Open File as writable + with open(filename, "w") as fd: + # Write each individual to file + for index, individual in enumerate(population): + if index == len(population) - 1: + fd.write(str(individual)) + else: + fd.write(str(individual) + "\n") + + +def load_generation(logger, filename): + """ + Loads strategies from a file + + Args: + logger (:obj:`logger.Logger`): A logger to log with + filename (str): Filename of file containing newline separated strategies + to read generation from + """ + population = [] + with open(filename) as file: + + # Read each individual from file + for individual in file: + strategy = actions.utils.parse(individual, logger) + population.append(strategy) + + return population + + +def initialize_population(logger, options, canary_id, disabled=None): + """ + Initializes the population from either random strategies or strategies + located in a file. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + options (dict): Options to respect in generating initial population. + Options that can be specified as keys: + + "load_from" (str, optional): File to load population from + population_size (int): Size of population to initialize + + "in-trees" (int): Number of trees to initialize in inbound forest + of each individual + + "out-trees" (int): Number of trees to initialize in outbound forest + of each individual + + "in-actions" (int): Number of actions to initialize in each + inbound tree of each individual + + "out-actions" (int): Number of actions to initialize in each + outbound tree of each individual + + "seed" (str): Strategy to seed this pool with + canary_id (str): ID of the canary strategy, used to associate each new + strategy with the packets captured during the canary phase + disabled (list, optional): List of actions that are disabled + + Returns: + list: New population of individuals + """ + + if options.get("load_from"): + # Load the population from a file + return load_generation(logger, options["load_from"]) + + # Generate random strategies + population = [] + + for _ in range(options["population_size"]): + p = generate_strategy(logger, options["in-trees"], options["out-trees"], options["in-actions"], + options["out-actions"], options["seed"], environment_id=canary_id, + disabled=disabled) + population.append(p) + + return population + + +def genetic_solve(logger, options, ga_evaluator): + """ + Run genetic algorithm with given options. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + options (dict): Options to respect. + ga_evaluator (:obj:`evaluator.Evaluator`): Evaluator to evaluate + strategies with + + Returns: + dict: Hall of fame of individuals + """ + # Directory to save off each generation so evolution can be resumed + ga_generations_dir = os.path.join(actions.utils.RUN_DIRECTORY, "generations") + + hall = {} + canary_id = None + if ga_evaluator and not options["no-canary"]: + canary_id = run_collection_phase(logger, ga_evaluator) + else: + logger.info("Skipping initial collection phase.") + + population = initialize_population(logger, options, canary_id, disabled=options["disable_action"]) + + try: + offspring = [] + elite_clones = [] + if options["seed"]: + elite_clones = [actions.utils.parse(options["seed"], logger)] + + # Evolution over given number of generations + for gen in range(options["num_generations"]): + # Debug printing + logger.info("="*(int(COLUMNS) - 25)) + logger.info("Generation %d:", gen) + + # Save current population pool + filename = os.path.join(ga_generations_dir, "generation" + str(gen) + ".txt") + write_generation(filename, population) + + # To store the best individuals of this generation to print + best_fit, best_ind = -10000, None + + # Mutation and crossover + offspring = mutation_crossover(logger, population, hall, options) + offspring += elite_clones + + # Calculate fitness + offspring = fitness_function(logger, offspring, ga_evaluator) + + total_fitness = 0 + # Iterate over the offspring to find the best individual for printing + for ind in offspring: + if ind.fitness is None and ga_evaluator: + logger.error("No fitness for individual found: %s.", str(ind)) + continue + total_fitness += ind.fitness + if ind.fitness is not None and ind.fitness >= best_fit: + best_fit = ind.fitness + best_ind = ind + + # Check if any individuals of this generation belong in the hall of fame + hall = add_to_hof(hall, offspring) + + # Save current hall of fame + filename = os.path.join(ga_generations_dir, "hall" + str(gen) + ".txt") + write_hall(filename, hall) + + # Status printing for this generation + logger.info("\nGeneration: %d | Unique Inviduals: %d | Avg Fitness: %d | Best Fitness [%s] %s: %s", + gen, get_unique_population_size(population), round(total_fitness / float(len(offspring)), 2), + best_ind.environment_id, str(best_fit), str(best_ind)) + + # Select next generation + population = selection_tournament(offspring, k=len(population) - options["elite_clones"], tournsize=10) + + # Add the elite clones + if options["elite_clones"] > 0: + elite_clones = [copy.deepcopy(best_ind) for x in range(options["elite_clones"])] + + # If the user interrupted, try to gracefully shutdown + except KeyboardInterrupt: + # Only need to stop the evaluator if one is defined + if ga_evaluator: + ga_evaluator.stop = True + logger.info("") + + finally: + if options["force_cleanup"]: + # Try to clean up any hanging docker containers/networks from the run + logger.warning("Cleaning up docker...") + try: + sp.check_call("docker stop $(docker ps -aq) > /dev/null 2>&1", shell=True) + except sp.CalledProcessError: + pass + + return hall + + +def collect_results(hall_of_fame): + """ + Collect final results from offspring. + + Args: + hall_of_fame (dict): Hall of fame of individuals + + Returns: + str: Formatted printout of the hall of fame + """ + # Sort first on number of runs, then by fitness. + best_inds = sorted(hall_of_fame, key=lambda ind: (len(hall_of_fame[ind]), sum(hall_of_fame[ind])/len(hall_of_fame[ind]))) + output = "Results: \n" + for ind in best_inds: + sind = str(ind) + output += "Avg. Fitness %s: %s (Evaluated %d times: %s)\n" % (sum(hall_of_fame[sind])/len(hall_of_fame[sind]), sind, len(hall_of_fame[sind]), hall_of_fame[sind]) + return output + + +def print_results(hall_of_fame, logger): + """ + Prints hall of fame. + + Args: + hall_of_fame (dict): Hall of fame to print + logger (:obj:`logging.Logger`): A logger to log results with + """ + logger.info("\n%s", collect_results(hall_of_fame)) + + +def write_hall(filename, hall_of_fame): + """ + Writes hall of fame out to a file. + + Args: + filename (str): Filename to write results to + hall_of_fame (dict): Hall of fame of individuals + """ + with open(filename, "w") as fd: + fd.write(collect_results(hall_of_fame)) + + +def eval_only(logger, requested, ga_evaluator, runs=1): + """ + Parses a string representation of a given strategy and runs it + through the evaluator. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + requested (str): String representation of requested strategy or filename + of strategies + ga_evaluator (:obj:`evaluator.Evaluator`): An evaluator to evaluate with + runs (int): Number of times each strategy should be evaluated + + Returns: + float: Success rate of tested strategies + """ + # The user can specify a file that contains strategies - check first if that is the case + if os.path.isfile(requested): + with open(requested, "r") as fd: + requested_strategies = fd.readlines() + if not requested_strategies: + logger.error("No strategies found in %s", requested) + return None + else: + requested_strategies = [requested] + # We want to override the client's default strategy retry logic to ensure + # we test to the number of runs requested + ga_evaluator.runs = 1 + population = [] + + for requested in requested_strategies: + for i in range(runs): + ind = actions.utils.parse(requested, logger) + population.append(ind) + logging.info("Computing fitness for: %s", str(ind)) + logging.info("\n%s", ind.pretty_print()) + + fits = [] + success = 0 + # Once the population has been parsed and built, test it + fitness_function(logger, population, ga_evaluator) + for strat in population: + fits.append(strat.fitness) + i = 0 + logger.info(fits) + for fitness in fits: + if fitness > 0: + success += 1 + logger.info("Trial %d: success! (fitness = %d)", i, fitness) + else: + logger.info("Trial %d: failure! (fitness = %d)", i, fitness) + i += 1 + if fits: + logger.info("Overall %d/%d = %d%%", success, i, int((float(success)/float(i)) * 100)) + logger.info("Exiting eval-only.") + return float(success)/float(i) + + +def restrict_headers(logger, protos, filter_fields, disabled_fields): + """ + Restricts which protocols/fields can be accessed by the algorithm. + + Args: + logger (:obj:`logging.Logger`): A logger to log with + protos (str): Comma separated string of protocols that are allowed + filter_fields (str): Comma separated string of fields to allow + disabled_fields (str): Comma separated string of fields to disable + """ + # Retrieve flag and protocol filters, and validate them + protos = protos.upper().split(",") + if filter_fields: + filter_fields = filter_fields.lower().split(",") + if disabled_fields: + disabled_fields = disabled_fields.split(",") + + actions.packet.Packet.restrict_fields(logger, protos, filter_fields, disabled_fields) + + +def driver(cmd): + """ + Main workflow driver for the solver. Parses flags and input data, and initiates solving. + + Args: + cmd (list): sys.argv or a list of arguments + + Returns: + dict: Hall of fame of individuals + """ + # Parse the given arguments + args = get_args(cmd) + + logger = setup_logger(args.log) + + lock_file_path = "/lock_file.txt" + if not args.no_lock_file and os.path.exists(lock_file_path): + logger.info("Lock file \"%s\" already exists.", lock_file_path) + return None + + try: + if not args.no_lock_file: + # Create lock file to prevent interference between multiple runs + open(lock_file_path, "w+") + + # Log the command run + logger.debug("Launching strategy evolution: %s", " ".join(cmd)) + logger.info("Logging results to %s", logger.ga_log_dir) + + if args.no_eval and args.eval_only: + print("ERROR: Cannot --eval-only with --no-eval.") + return None + + requested_strategy = args.eval_only + + # Define an evaluator for this session + ga_evaluator = None + if not args.no_eval: + cmd += ["--output-directory", actions.utils.RUN_DIRECTORY] + ga_evaluator = evaluator.Evaluator(cmd, logger) + + # Check if the user only wanted to evaluate a single given strategy + # If so, evaluate it, and exit + if requested_strategy or requested_strategy == "": + # Disable evaluator empty strategy skipping + ga_evaluator.skip_empty = False + eval_only(logger, requested_strategy, ga_evaluator, runs=args.runs) + return None + + restrict_headers(logger, args.protos, args.fields, args.disable_fields) + + actions.trigger.GAS_ENABLED = (not args.no_gas) + if args.fix_trigger: + actions.trigger.FIXED_TRIGGER = actions.trigger.Trigger.parse(args.fix_trigger) + + requested_seed = args.seed + if requested_seed or requested_seed == "": + try: + requested_seed = actions.utils.parse(args.seed, logger) + except (TypeError, AssertionError, actions.tree.ActionTreeParseError): + logger.error("Failed to parse given strategy: %s", requested_seed) + raise + + # Record all of the options supplied by the user to pass to the GA + options = {} + options["no_reject_empty"] = not args.no_reject_empty + options["population_size"] = args.population + options["out-trees"] = args.out_trees + options["in-trees"] = args.in_trees + options["in-actions"] = args.in_actions + options["out-actions"] = args.out_actions + options["force_cleanup"] = args.force_cleanup + options["num_generations"] = args.generations + options["seed"] = args.seed + options["elite_clones"] = args.elite_clones + options["allowed_retries"] = args.allowed_retries + options["mutation_pb"] = args.mutation_pb + options["crossover_pb"] = args.crossover_pb + options["no-canary"] = args.no_canary + options["load_from"] = args.load_from + + disable_action = [] + if args.disable_action: + disable_action = args.disable_action.split(",") + options["disable_action"] = disable_action + + logger.info("Initializing %d strategies with %d input-action trees and %d output-action trees of input size %d and output size %d for evolution over %d generations.", + args.population, args.in_trees, args.out_trees, args.in_actions, args.out_actions, args.generations) + + hall_of_fame = {} + try: + # Kick off the main genetic driver + hall_of_fame = genetic_solve(logger, options, ga_evaluator) + except KeyboardInterrupt: + logger.info("User shutdown requested.") + if ga_evaluator: + ga_evaluator.shutdown() + + if hall_of_fame and not args.no_print_hall: + # Print the final results + print_results(hall_of_fame, logger) + + # Teardown the evaluator if needed + if ga_evaluator: + ga_evaluator.shutdown() + finally: + # Remove lock file + if os.path.exists(lock_file_path): + os.remove(lock_file_path) + return hall_of_fame + + +if __name__ == "__main__": + driver(sys.argv[1:]) diff --git a/library.py b/library.py new file mode 100644 index 0000000..c043c72 --- /dev/null +++ b/library.py @@ -0,0 +1,480 @@ +# The following strategies have been learned as successful against the Great Firewall. +WORKING_STRATEGIES = [ + { + "strategy" : "\/", + "success_rate" : .03, + "description" : "No strategy", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + + # TCB Desync - High DataOfs + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:chksum:replace:25776},),)-", + "success_rate" : .98, + "description" : "TCP Desync - Increment Dataofs - Corrupt Chksum", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{IP:ttl:replace:10},),)-", + "success_rate" : .98, + "description" : "TCP Desync - Increment Dataofs - Small TTL", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:flags:replace:FRAPUN},),)-", + "success_rate" : .26, + "description" : "TCP Desync - Increment Dataofs - Invalid Flags", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:10}(tamper{TCP:ack:corrupt},),)-", + "success_rate" : .94, + "description" : "TCP Desync - Increment Dataofs - Corrupt ACK", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:options-wscale:corrupt}(tamper{TCP:dataofs:replace:8},),)-", + "success_rate" : .98, + "description" : "TCP Desync - Increment Dataofs - Corrupt WScale", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + # TCB Desync - Load corruption + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:chksum:corrupt},),)-", + "success_rate" : .98, + "description" : "TCP Desync - Invalid Payload - Corrupt Chksum", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{IP:ttl:replace:8}(duplicate(fragment{tcp:-1:False},),),),)-", + "success_rate" : .98, + "description" : "TCP Desync - Invalid Payload - Small TTL", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:load:corrupt}(tamper{TCP:ack:corrupt}(duplicate(fragment{tcp:-1:False},),),),)-", + "success_rate" : .93, + "description" : "TCP Desync - Invalid Payload - Corrupt ACK", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + + # TCB Teardown (with RST) + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},))-", + "success_rate" : .95, + "description" : "TCB Teardown - with RST - Corrupt Chksum, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-", + "success_rate" : .51, + "description" : "TCB Teardown - with RST - Corrupt Chksum, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:10},))-", + "success_rate" : .87, + "description" : "TCB Teardown - with RST - Small TTL, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{IP:ttl:replace:9},),)-", + "success_rate" : .52, + "description" : "TCB Teardown - with RST - Small TTL, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-", + "success_rate" : .86, + "description" : "TCB Teardown - with RST - Invalid md5Header, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:RA},),)-", + "success_rate" : .44, + "description" : "TCB Teardown - with RST - Invalid md5Header, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + # Teardown with RST/ACK + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:replace:27925},))-", + "success_rate" : .90, + "description" : "TCB Teardown - with RST/ACK - Corrupt Chksum, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{TCP:chksum:replace:27925},),)-", + "success_rate" : .66, + "description" : "TCB Teardown - with RST/ACK - Corrupt Chksum, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},))-", + "success_rate" : .94, + "description" : "TCB Teardown - with RST/ACK - Small TTL, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{IP:ttl:replace:10},),)-", + "success_rate" : .57, + "description" : "TCB Teardown - with RST/ACK - Small TTL, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},))-", + "success_rate" : .94, + "description" : "TCB Teardown - with RST/ACK - Invalid md5Header, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:options-md5header:corrupt}(tamper{TCP:flags:replace:R},),)-", + "success_rate" : .48, + "description" : "TCB Teardown - with RST/ACK - Invalid md5Header, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:RA}(tamper{TCP:ack:corrupt},),)-", + "success_rate" : .43, + "description" : "TCB Teardown - with RST/ACK - Corrupt ACK, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:RA}(tamper{TCP:ack:corrupt},))-", + "success_rate" : .31, + "description" : "TCB Teardown - with RST/ACK - Corrupt ACK, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + # TCB Teardown w/ Invalid Flags + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},))- ", + "success_rate" : .89, + "description" : "TCB Teardown - Invalid Flags - Corrupt Chksum, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:chksum:corrupt},),)-", + "success_rate" : .48, + "description" : "TCB Teardown - Invalid Flags - Corrupt Chksum, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FREACN}(tamper{IP:ttl:replace:10},))-", + "success_rate" : .96, + "description" : "TCB Teardown - Invalid Flags - Small TTL, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{IP:ttl:replace:10},),)-", + "success_rate" : .56, + "description" : "TCB Teardown - Invalid Flags - Small TTL, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(,tamper{TCP:flags:replace:FRAPUN}(tamper{TCP:options-md5header:corrupt},))-", + "success_rate" : .94, + "description" : "TCB Teardown - Invalid Flags - Invalid md5Header, High Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:FRAPUEN}(tamper{TCP:options-md5header:corrupt},),)-", + "success_rate" : .55, + "description" : "TCB Teardown - Invalid Flags - Invalid md5Header, Low Percentage", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:A]-tamper{TCP:load:replace:a5z2}(tamper{TCP:flags:replace:CN},)-", + "success_rate" : .00, + "description" : "TCB Teardown - Invalid Flags - Invalid Payload", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + # Segmentation + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:8:False}-|[TCP:flags:A]-tamper{TCP:seq:replace:2258679050}-", + "success_rate" : .92, + "description" : "Segmentation - With ACK - Offsets, subvariant 1", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:8:False}-| [TCP:flags:A]-tamper{TCP:seq:replace:2258679050}(tamper{TCP:flags:replace:PA},)-", + "success_rate" : .95, + "description" : "Segmentation - With ACK - Offsets, subvariant 2", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:8:True}(,fragment{tcp:4:True})-", + "success_rate" : .98, + "description" : "Segmentation - Reassembly - Offsets, subvariant 1", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:4:True}(,fragment{tcp:19:True})-", + "success_rate" : .96, + "description" : "Segmentation - Reassembly - Offsets, subvariant 2", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + # Hybrid Strategies + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:68},),)-| \/", + "success_rate" : .53, + "description" : "Hybrid Strategy - HTTP - Fragment and set Fin flag", + "test_type" : "http", + "server_side" : False, + "country" : "china" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:flags:replace:F}(tamper{IP:len:replace:68},),)-| \/", + "success_rate" : 1, + "description" : "Hybrid Strategy - DNS Over TCP - Fragment and set Fin flag", + "test_type" : "dns_tcp", + "server_side" : False, + "country" : "china" + }, + # Server-side strategies + { + "strategy" : "[TCP:flags:SA]-duplicate(tamper{TCP:flags:replace:FREC}(duplicate,),tamper{TCP:flags:replace:FSEC})-| \/", + "success_rate" : .43, + "description" : "Server-Side - Simultaneous Open", + "test_type" : "http", + "server_side" : True, + "country" : "china" + }, + # DNS over UDP strategies + { + "strategy" : "\/ [UDP:sport:53:2]-drop-", + "success_rate" : .96, + "description" : "DNS - Drop first two bad responses", + "test_type" : "dns", + "server_side" : False, + "country" : "china" + }, + # India HTTP strategies + { + "strategy" : "[TCP:options-altchksum:]-tamper{TCP:options-uto:corrupt}(duplicate,)-| \/", + "success_rate" : 1, + "description" : "Invalid Options", + "test_type" : "http", + "server_side" : False, + "country" : "india" + }, + { + "strategy" : "[TCP:options-mss:]-tamper{TCP:options-md5header:corrupt}-| \/", + "success_rate" : 1, + "description" : "Invalid Options", + "test_type" : "http", + "server_side" : False, + "country" : "india" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{TCP:dataofs:replace:9},)-| \/", + "success_rate" : 1, + "description" : "Increasing dataofs", + "test_type" : "http", + "server_side" : False, + "country" : "india" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{IP:len:replace:64},)-|", + "success_rate" : 1, + "description" : "IP length", + "test_type" : "http", + "server_side" : False, + "country" : "india" + }, + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:-1:True}-|", + "success_rate" : 1, + "description" : "Segmentation", + "test_type" : "http", + "server_side" : False, + "country" : "india" + }, + { + "strategy" : "[TCP:flags:SA]-tamper{TCP:window:replace:98}-|", + "success_rate" : 1, + "description" : "Server side", + "test_type" : "http", + "server_side" : True, + "country" : "india" + }, + { + "strategy" : "[TCP:flags:PA]-fragment{tcp:-1:True}-|", + "success_rate" : 1, + "description" : "Segmentation", + "test_type" : "kazakhstan_injected_https", + "server_side" : False, + "country" : "kazakhstan" + }, + { + "strategy" : "[TCP:flags:PA]-duplicate(tamper{IP:len:replace:78},)-|", + "success_rate" : 1, + "description" : "Segmentation Exploit - Small IP length", + "test_type" : "kazakhstan_injected_https", + "server_side" : False, + "country" : "kazakhstan" + }, + { + "strategy" : "[TCP:flags:S]-duplicate(,tamper{TCP:load:corrupt})-|", + "success_rate" : 1, + "description" : "Desync: load on second SYN", + "test_type" : "kazakhstan_injected_https", + "server_side" : False, + "country" : "kazakhstan" + }, + { + "strategy" : "[TCP:flags:S]-duplicate(tamper{TCP:flags:replace:SA},)-|", + "success_rate" : 1, + "description" : "TCB Turnaround", + "test_type" : "kazakhstan_injected_https", + "server_side" : False, + "country" : "kazakhstan" + }, +] + +# Strategies that evade the lab censors +LAB_STRATEGIES = [ + { + "strategy" : "[TCP:flags:A]-tamper{TCP:flags:replace:F}-| \/", + "censors" : ["censor6", "censor7", "censor8"], + "description" : "Interrupts the 3-way handshake with a FIN. The server ignores the FIN, as \ + the 3-way handshake has not been done yet, so there is no connection to teardown. \ + Since the client's very next packet is a PSH-ACK, the ACK in this packet serves to \ + complete the 3-way handshake, but the censor tears down it's TCB at the FIN." + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:seq:corrupt}(tamper{TCP:flags:replace:R},),)-| \/", + "censors" : ["censor6", "censor7"], + "description" : "Tears down a TCB right after a 3-way handshake by injecting a RST packet." + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/", + "censors" : ["censor6", "censor7", "censor8", "censor8b"], + "description" : "Creates a RST injection packet to tear down a TCB and corrupts the checksum, exploiting a censor \ + that does not validate packet checksums." + }, + { + "strategy" : "\/ [TCP:dataofs:5]-drop-|", + "censors" : ["censor2"], + "description" : "RST Packets (at least those constructed with scapy) have a data offset of 5. This \ + beats a censor that only sends RSTs to the client by dropping the RST packets." + }, + { + "strategy" : "\/ [TCP:flags:R]-drop-|", + "censors" : ["censor2"], + "description" : "Beats a censor that only sends RSTs to the client by dropping the RST packets." + }, + { + "strategy" : "\/ [TCP:window:8192]-drop-|", + "censors" : ["censor2"], + "description" : "RST Packets (at least those constructed with scapy) have a default window size of 8192. This \ + beats a censor that only sends RSTs to the client by dropping the RST packets." + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:replace:14239},),duplicate(tamper{TCP:flags:replace:S}(tamper{TCP:chksum:replace:14239},),))-| \/", + "censors" : ["censor1", "censor2", "censor3", "censor5", "censor6", "censor7", "censor8", "censor8b", "censor9"], + "description" : "Triggers a RST packet and SYN packet to be sent immediately after the 3-way handshake \ + finishes. The server ignores the RST, as the chksum is corrupted, and ignores the SYN, \ + as a connection is already up. The censor sees the RST and enters the resynchronization \ + state, and the immediate follow-up SYN packet causes the TCB to be deschronized from the \ + real connection." + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:replace:15239},),duplicate(tamper{TCP:flags:replace:S}(tamper{TCP:seq:corrupt}(tamper{TCP:chksum:corrupt},),),))-| \/", + + "censors" : ["censor1", "censor2", "censor3", "censor5", "censor6", "censor7", "censor8", "censor8b", "censor9"], + "description" : "Triggers a RST packet and SYN packet to be sent immediately after the 3-way handshake \ + finishes. The server ignores the RST, as the chksum is corrupted, and ignores the SYN, \ + as a connection is already up. The censor sees the RST and enters the resynchronization \ + state, and the immediate follow-up SYN packet with a new seq causes the TCB to be deschronized from the \ + real connection." + }, + + { + "strategy" : "[TCP:flags:A]-tamper{TCP:dataofs:replace:0}-| \/", + "censors" : ["censor1", "censor2", "censor3", "censor5", "censor9", "censor10"], + "description" : "The dataofs field in the TCP header tells applications where the payload of the packet \ + starts. By replacing the dataofs to 0 on a packet without a payload (ACK), it makes the \ + TCP header look like data. Servers ignore this, but a censor that is trying to keep a TCB \ + synchronized will be desynchronized from the connection when it gets a payload of an incorrect \ + length." + + }, + { + "strategy" : "[TCP:flags:A]-duplicate(tamper{TCP:dataofs:replace:0},)-| \/", + "censors" : ["censor1", "censor2", "censor3", "censor5", "censor9", "censor10"], + "description" : "The dataofs field in the TCP header tells applications where the payload of the packet \ + starts. By replacing the dataofs to 0 on a packet without a payload (ACK), it makes the \ + TCP header look like data. Servers ignore this, but a censor that is trying to keep a TCB \ + synchronized will be desynchronized from the connection when it gets a payload of an incorrect \ + length. This strategy is functionally equivalent to the above strategy, but also preserves the \ + original packet." + + }, +] diff --git a/plugins/discard/client.py b/plugins/discard/client.py new file mode 100644 index 0000000..a2d4c68 --- /dev/null +++ b/plugins/discard/client.py @@ -0,0 +1,171 @@ +""" +Client + +Run by the evaluator, sends data to discard server. + +Not usually used for training because in Python it is difficult to distinguish between a successful +strategy and an unsuccessful strategy. This is because in the discard protocol (in a good case), +the client will send data and the server will throw it away (but ACK it). In a bad case (such as a +failing strategy that breaks the TCP connection), the client sends the data but it does not reach +the server (so it is not ACKed). However, in Python, it is non trivial to distinguish these two +cases, as neither send() nor sendall() will raise a timeout or check if the data that is sent is +ACKed. +""" + +import argparse +import logging +import os +import random +import socket +import sys +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(10) + +import actions.utils + +from plugins.plugin_client import ClientPlugin + +import signal + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +# Sets up timeout signal because currently, sendall does not give a timeout which is making the +# client believe that the connection has not been torn down. +class TimeoutError(Exception): + """ + Houses a TimeoutError so we can cut off sendall. + """ + pass + +def handle_timeout(signum, frame): + import errno + raise TimeoutError(os.strerror(errno.ETIME)) + + +class DiscardClient(ClientPlugin): + """ + Defines the Discard client. + """ + name = "discard" + + def __init__(self, args): + """ + Initializes the discard client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='Discard Client') + + # If we know whether the censor we are training against injects content, we can optimize the plugin's behavior. + # Censors that inject content will give us a very clear signal - after we send content, we can use `recv()` to get + # the response from the censor. Since this is the discard protocol, if we ever receive data, this is from the censor. + parser.add_argument('--injection-censor', action='store_true', help="whether this censor injects content or sends RSTs to censor") + parser.add_argument('--server', action='store', help="server to connect to") + # Makes it easier to craft fake HTTP requests to trip censorship + parser.add_argument('--http-request', action='store', help="send an HTTP get request with the given hostname to the discard server") + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden GET request to the server. + """ + fitness = 0 + port = int(args["port"]) + server = args["server"] + bad_word = args["bad_word"] + msg = bad_word + injection_censor = args.get("injection_censor") + if args.get("http_request"): + msg = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % args.get("http_request") + try: + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect((server, port)) + for idx in range(0, 5): + if type(msg) == str: + msg = msg.encode() + + # Set a 10 second timeout on the socket. Timeouts do not interrupt send() or sendall(). + client.settimeout(10) + + # Manually create a 5 second timeout + timeout = 5 + + # Setup the timeout as a signal alarm + signal.signal(signal.SIGALRM, handle_timeout) + signal.alarm(timeout) + + reached_timeout = False + try: + client.send(msg) + # Give the alarm time to realize it must go off + time.sleep(1) + except TimeoutError: + logger.debug("sendall() timed out") + fitness -= 100 + reached_timeout = True + finally: + signal.alarm(0) + + # If the censor injects content, checks to make sure nothing is sent back from the server. + # If the recv times out, then the procedure was successful. If injected content is sent back, + # censorship has occurred. + if injection_censor: + try: + server_data = client.recv(1024) + logger.debug("Data received: %s", server_data.decode('utf-8', 'ignore')) + fitness -= 90 + except socket.timeout: + fitness += 100 + logger.debug("No data received from a censor.") + # If the censor is not an injection censor and the connection is not teared down, the strategy + # is successful. + else: + if idx != 0 and not reached_timeout: + fitness += 90 + client.close() + # If the fitness is 0, the strategy did something to corrupt/interfere with the socket + # sending/receiving, usually by just artificially closing the connection. This behavior + # should not be rewarded with a higher fitness + if fitness == 0: + fitness -= 100 + + except socket.error as exc: + # If the censor we're running against tears down connects via RSTs, we can punish RSTs as + # if the strategy did not harm the underlying connection. However, if the censor only injects + # traffic, not resets, we should punish RSTs harshly, as the strategy likely caused it. + + if exc.errno == 104: + if injection_censor: + fitness -= 110 + else: + fitness -= 100 + logger.debug("Client: Connection RST.") + else: + fitness -= 100 + logger.exception("Socket error caught in client discard test.") + except Exception: + logger.exception("Exception caught in client discard test.") + fitness = -120 + finally: + logger.debug("Client finished discard test.") + signal.alarm(0) + return fitness * 4 + diff --git a/plugins/discard/server.py b/plugins/discard/server.py new file mode 100644 index 0000000..50cb7ab --- /dev/null +++ b/plugins/discard/server.py @@ -0,0 +1,96 @@ +import argparse +import os +import socket +import subprocess + +from plugins.plugin_server import ServerPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class DiscardServer(ServerPlugin): + """ + Defines the Discard client. + """ + name = "discard" + def __init__(self, args): + """ + Initializes the client. + """ + ServerPlugin.__init__(self) + self.args = args + if args: + self.port = args["port"] + + @staticmethod + def get_args(command): + """ + Defines arguments for this plugin + """ + super_args = ServerPlugin.get_args(command) + + parser = argparse.ArgumentParser(description='Discard Server') + + args, _ = parser.parse_known_args(command) + args = vars(args) + super_args.update(args) + return super_args + + def run(self, args, logger): + """ + Initializes the Discard server. + """ + logger.debug("Discard server initializing") + try: + port = int(args["port"]) + control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Allow socket re-use + control_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_address = ('0.0.0.0', port) + logger.debug("Binding to server address 0.0.0.0:%d" % port) + control_socket.bind(server_address) + control_socket.settimeout(5) + control_socket.listen(1) + except: + logger.exception("Caught exception in discard run") + return + + try: + connection, client_address = self.get_request(control_socket) + if not connection: + logger.error("Failed to get connection") + return + for i in range(0, 5): + data = connection.recv(1024) + + connection.close() + except socket.error as e: + if e.errno == 104: + logger.debug("Server: Connection RST.") + else: + logger.debug("Server: Client quit.") + except socket.ConnectionResetError: + logger.debug("Server: Connection RST.") + except Exception: + logger.exception("Failed during server communication.") + finally: + logger.debug("Server exiting") + + def get_request(self, control_socket): + """ + Get a request from the socket. + """ + while True: + try: + sock, addr = control_socket.accept() + sock.settimeout(5) + return (sock, addr) + except socket.timeout: + pass + return (None, None) + + def stop(self): + """ + Stops this server. + """ + ServerPlugin.stop(self) diff --git a/plugins/dns/client.py b/plugins/dns/client.py new file mode 100644 index 0000000..ca2f207 --- /dev/null +++ b/plugins/dns/client.py @@ -0,0 +1,122 @@ +""" +Client + +Run by the evaluator, tries to make a GET request to a given server +""" + +import argparse +import logging +import os +import random +import socket +import sys +import time +import traceback +import urllib.request + +import dns.resolver +import requests + +import actions.utils +from plugins.plugin_client import ClientPlugin + + +class DNSClient(ClientPlugin): + """ + Defines the DNS client. + """ + name = "dns" + + def __init__(self, args): + """ + Initializes the DNS client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='DNS Client') + + parser.add_argument('--use-tcp', action='store_true', help='leverage TCP for this plugin') + parser.add_argument('--dns-server', action='store', default="8.8.8.8", help='domain server to connect to') + parser.add_argument('--query', action='store', default="facebook.com", help='censored domain to query') + parser.add_argument('--timeout', action='store', default="3", type=int, help='how long in seconds the client should wait for a response') + parser.add_argument('--port', action='store', default="53", type=int, help='port the DNS server is running on (must be 53)') + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden DNS query. + """ + fitness = 0 + to_lookup = args.get("query", "facebook.com") + dns_server = args.get("dns_server", "8.8.8.8") + use_tcp = args.get("use_tcp", False) + assert dns_server, "Cannot launch DNS test with no DNS server" + assert to_lookup, "Cannot launch DNS test with no server to query" + fitness = -1000 + try: + fitness = self.dns_test(to_lookup, dns_server, args["output_directory"], args["environment_id"], logger, timeout=args.get("timeout", 3), use_tcp=use_tcp) + except Exception: + logger.exception("Exception caught in DNS test to resolver %s.", dns_server) + fitness += -100 + + # When performing a DNS test, a timeout is indistinguishable from + # a reset, which means we can't tell if the strategy broke the packet + # stream, or if the censor caught us. Strategies that break the stream + # should be punished more harshly, so raise the fitness slightly + # if the engine detected censorship for failed DNS tests. + if use_tcp and engine and engine.censorship_detected and fitness < 0: + fitness += 10 + return fitness * 4 + + def dns_test(self, to_lookup, dns_server, output_dir, environment_id, logger, timeout=3, use_tcp=False): + """ + Makes a DNS query to a given censored domain. + """ + # Make the path an absolute path + if not output_dir.startswith("/"): + output_dir = os.path.join(actions.utils.PROJECT_ROOT, output_dir) + + resolver = dns.resolver.Resolver() + protocol = "UDP" + if use_tcp: + protocol = "TCP" + + logger.debug("Querying %s to DNS server %s over %s" % (to_lookup, dns_server, protocol)) + resolver.nameservers = [dns_server] + # Setup the timeout and lifetime for this resolver + resolver.timeout = timeout + resolver.lifetime = 3 + + try: + answer = resolver.query(to_lookup, "A", tcp=use_tcp)[0] + logger.debug("Got IP address: %s" % answer) + # At this point, we've been given an IP address by the DNS resolver, but we don't + # yet know if this IP address is a bogus injected response, or legitimate. Further, + # because we are likely running this code from within a censored regime which might + # employ secondary censorship at the IP level, we cannot check if this IP is legit + # here. Instead, we write it out to a file for the evaluator to extract and check for us. + with open(os.path.join(output_dir, "flags", environment_id)+".dnsresult", "w") as dnsfile: + dnsfile.write(str(answer)) + # For now, set fitness to a positive metric, though the evaluator will lower it if + # the IP address we were given was bogus. + fitness = 100 + except dns.exception.Timeout: + logger.error("DNS query timed out.") + fitness = -100 + except dns.resolver.NoNameservers: + logger.error("DNS server failed to respond") + fitness = -100 + + return fitness diff --git a/plugins/dns/plugin.py b/plugins/dns/plugin.py new file mode 100644 index 0000000..d82484b --- /dev/null +++ b/plugins/dns/plugin.py @@ -0,0 +1,143 @@ +""" +DNS Plugin driver + +Overrides the default evaluator plugin handling so we can check for legit IPs for UDP tests. +""" + +import argparse +import calendar +import copy +import logging +import os +import random +import socket +import sys +import tempfile +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import actions.utils + +from plugins.plugin import Plugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH)) + + +class DNSPluginRunner(Plugin): + """ + Defines the DNS plugin runner. + """ + name = "dns" + + def __init__(self, args): + """ + Marks this plugin as enabled + """ + self.enabled = True + + def check_legit_ip(self, ip, logger, domain="facebook"): + """ + Helper method to check if the given IP address is serving web content. + """ + url = "http://%s" % ip + logger.debug("Checking %s if returned legitimate %s" % (url, domain)) + try: + res = requests.get(url, allow_redirects=False, timeout=3) + if res.status_code == 400: + res.raise_for_status() + # If we got a 301 redirect, the res.text will be empty, but facebook will show up in + # the headers + for header in res.headers: + if domain in res.headers[header]: + return True + # Otherwise, check the res.text + return domain in res.text + except Exception as exc: + logger.debug("Exception caught in checking DNS result %s: %s", url, exc) + return False + + def start(self, args, evaluator, environment, ind, logger): + """ + Runs the plugins + """ + # Start the server + port = args.get("port", 53) + use_tcp = evaluator.client_args.get("use_tcp", False) + + if port != 53: + logger.warning("Warning: Given port %s, but GFW only censors on port 53.", str(port)) + + # Disable wait for server - it checks based on binding to a TCP port + evaluator.server_args.update({"no_wait_for_server" : True}) + + # If we're given a server to start, start it now + if evaluator.server_cls and not args.get("external_server"): + # If a test using TCP has been requested, switch the server to that mode + if use_tcp: + evaluator.server_args.update({"listener": "socket_TCP"}) + server = evaluator.start_server(evaluator.server_args, environment, logger) + evaluator.client_args.update({"dns_server": evaluator.args["server"]}) + + fitness = evaluator.run_client(evaluator.client_args, environment, logger) + + if evaluator.server_cls and not evaluator.args["external_server"]: + evaluator.stop_server(environment, server) + + evaluator.read_fitness(ind) + + # If the engine ran on the server side, ask that it punish fitness + if evaluator.args["server_side"]: + ind.fitness = server.punish_fitness(ind.fitness, logger) + # When performing a DNS test, a timeout is indistinguishable from + # a reset, which means we can't tell if the strategy broke the packet + # stream, or if the censor caught us. Strategies that break the stream + # should be punished more harshly, so raise the fitness slightly + # if the engine detected censorship for failed DNS tests. + if use_tcp and server.engine and server.engine.censorship_detected and ind.fitness < 0: + logger.debug("Censorship detected - adjusting positively for not killing stream") + ind.fitness += 40 + + output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory")) + fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness" + with open(fitpath, "w") as fitfile: + fitfile.write(str(ind.fitness)) + + if evaluator.args["external_client"]: + command = 'cat %s/%s/%s/%s.dnsresult' % (environment["worker"]["geneva_path"], evaluator.args["output_directory"], actions.utils.FLAGFOLDER, environment["id"]) + dns_result, error_lines = evaluator.remote_exec_cmd(environment["remote"], command, logger) + if not dns_result: + logger.debug("Failed to get DNS result.") + else: + result = dns_result[0] + logger.debug("Got result: %s" % result) + # If the IP we got back was bad, we must fail the strategy + if not self.check_legit_ip(result, logger, domain="facebook"): + ind.fitness = -360 + output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory")) + fitpath = os.path.join(PROJECT_ROOT, output_path, actions.utils.FLAGFOLDER, environment["id"]) + ".fitness" + with open(fitpath, "w") as fitfile: + fitfile.write(str(ind.fitness)) + + # Log the fitness + #logger.info("[%s] Fitness %s: %s" % (ind.environment_id, str(ind.fitness), str(ind))) + + return ind.environment_id, ind.fitness + + @staticmethod + def get_args(command): + """ + Defines required global args for this plugin + """ + parser = argparse.ArgumentParser(description='DNS plugin runner', allow_abbrev=False) + parser.add_argument('--use-tcp', action='store_true', help='leverage TCP for this plugin') + parser.add_argument('--environment-id', action='store', help="ID of the current environment") + parser.add_argument('--output-directory', action='store', help="Where to output results") + parser.add_argument('--port', action='store', type=int, default=53, help='port to use') + args, _ = parser.parse_known_args(command) + return vars(args) diff --git a/plugins/dns/server.py b/plugins/dns/server.py new file mode 100644 index 0000000..633dabe --- /dev/null +++ b/plugins/dns/server.py @@ -0,0 +1,463 @@ +""" +Code influenced from: +- https://github.com/emileaben/scapy-dns-ninja/blob/master/dns-ninja-server.py +- https://thepacketgeek.com/scapy-p-09-scapy-and-dns/ + +""" + +import argparse + +# DNS Modules +import dns.zone +from dns.exception import DNSException + +# Scapy modules +from scapy.layers.dns import * +from scapy.all import send + +# Debugging +from pprint import pformat + +# TLDs +from tld import get_fld + +import inspect +import random +import os +import sys + +from plugins.plugin_server import ServerPlugin + +# Listener - NetfilterQueue +try: + from netfilterqueue import NetfilterQueue +except ImportError: + print("ERROR: Failed to import netfilerqueue.") + +# Listener - Socket +import socket + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + +# Utils +import datetime +import actions.utils +import logging + +# Default values +INTERFACE = "lo0" +LISTENER = "socket_UDP" +PORT = 53 +DNS_RESOLVER = "1.1.1.1" +ZONES_DIR = "zones/" +LOG_LEVEL = "info" + + +class DNSServer(ServerPlugin): + """ + Purpose: Handle incoming DNS queries and respond with resource records defined in a zone configuration file (if + exists for that domain) or respond with the answer given by a DNS resolver + + Features: + - Loads zone configuration files (--zones-dir) + - Forwards DNS requests to a DNS resolver for domains that it does not know the answer to (--dns-resolver) + - DNS forwarding can be disabled with (--no-forwarding) + - Can act as the authority server for all DNS responses + + Zones: + - Support for A, MX, NS, TXT and CNAME + - Other records may be automatically supported through the default action (no special case) + - Only the first string per TXT record will be retrieved to avoid duplicated quotes + + Logging: + - Logs are created for each run and saved in the directory specified (--log-dir) + - Logs can be disabled with (--no-log) + + Python Test: tests/test_dns_server.py + """ + name = "dns" + netfilter_queue = 'netfilterqueue' + socket_UDP = 'socket_UDP' + socket_TCP = 'socket_TCP' + + def __init__(self, args, logger=None): + """ + Initializes the DNS Server. + """ + ServerPlugin.__init__(self) + self.nfqueue = None + self.nfqueue_num = None + self.sock = None + self.running = False + self.zones = {} + self.packet_counter = 0 + self.logger = logger + + if not args: + return + + # Arguments + self.interface = args["interface"] + self.listener = args["listener"] + self.port = args["port"] + self.authority = args["authority"] + self.resolver = args["dns_resolver"] + self.zones_dir = args["zones_dir"] + + def get_args(command): + """ + Sets up argparse and collects arguments. + """ + super_args = ServerPlugin.get_args(command) + + parser = argparse.ArgumentParser(description='DNS Server') + + # Network Configuration + parser.add_argument('--interface', action='store', help='Interface to listen on', default=INTERFACE) + parser.add_argument('--listener', action='store', choices=(DNSServer.socket_TCP, DNSServer.socket_UDP, + DNSServer.netfilter_queue), + help='Set the listener (Netfilterqueue is linux only)', default=DNSServer.socket_UDP) + parser.add_argument('--port', type=int, action='store', help='DNS Server port to listen on', default=PORT) + + # Zones + parser.add_argument("--zones-dir", action='store', help="Zones directory", default=ZONES_DIR) + + # Authority + parser.add_argument('--authority', action='store_true', help='States that the DNS server is the authority server of' + ' all DNS responses') + # DNS Resolver + parser.add_argument('--dns-resolver', action='store', help="Specify a DNS resolver to forward DNS queries", + default=DNS_RESOLVER) + parser.add_argument('--no-forwarding', action='store_true', help='Disable forwarding DNS queries to a DNS resolver', + default=False) + + parser.add_argument('--log', action='store', choices=("debug", "info", "error"), help="Sets the log level", + default=LOG_LEVEL) + args, _ = parser.parse_known_args(command) + args = vars(args) + super_args.update(args) + return super_args + + def run(self, args, logger): + """ + Starts the DNS Service + """ + self.running = True + self.logger = logger + # Setup the Listener + if self.listener == DNSServer.netfilter_queue: # Netfilter Queue + self.nfqueue_num = random.randint(11, 255) + os.system( + 'iptables -t mangle -A PREROUTING -p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num) + self.nfqueue = NetfilterQueue() + self.nfqueue.bind(self.nfqueue_num, self.process_packet_netfilter) + + elif self.listener == DNSServer.socket_UDP: # UDP Socket + try: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_UDP) + self.sock.bind(('0.0.0.0', self.port)) + except socket.error as err: + raise Exception("Error opening UDP socket") + elif self.listener == DNSServer.socket_TCP: # TCP Socket + try: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind(('0.0.0.0', self.port)) + except Exception as err: + raise Exception("Error opening TCP socket") + else: # None selected + raise Exception("No listener has been selected") + + # Load the DNS zones this server will support + self.load_zones() + self.logger.debug("Starting the DNS service") + self.write_startup_file(args, logger) + + # Netfilter + if self.listener == DNSServer.netfilter_queue: + try: + self.nfqueue.run() + except KeyboardInterrupt: + os.system('iptables -t mangle -D PREROUTING ' + '-p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num) + + # Socket UDP + elif self.listener == DNSServer.socket_UDP: + while True: + try: + data = self.sock.recv(1024) + except socket.timeout: + continue + response_packet = self.build_response_packet(data) + + if response_packet is not None: + send(response_packet, verbose=0)#, iface=self.interface) + + # Socket TCP + elif self.listener == DNSServer.socket_TCP: + self.sock.listen(10000) + self.logger.debug("Socket is listening") + + # Continuously accept new connections + while True: + try: + connection, addr = self.sock.accept() + + # Two byte length field + message_length = connection.recv(2) + if not message_length: + continue + + message_length = int(message_length[0]) * 256 + int(message_length[1]) + + # Receive the DNS contents + dns_contents = connection.recv(message_length) + if not dns_contents: + continue + + # Build response + response_packet = self.build_response_packet(dns_contents, False) + + if response_packet is not None: + length = len(response_packet) + connection.send(length.to_bytes(2, byteorder='big') + raw(response_packet)) + + connection.close() + except KeyboardInterrupt: + self.sock.close() + break + except Exception: + pass + + def load_zones(self): + """ + Loads the DNS Zones in the zones directory specified (zones_dir) + """ + zones_dir = os.path.join(BASEPATH, self.zones_dir) + self.logger.debug("Loading the DNS zones from %s", zones_dir) + + # Each file in the zones directory is a domain + for domain in os.listdir(zones_dir): + try: + self.zones[domain] = dns.zone.from_file(zones_dir + domain, domain, rdclass=1, relativize=False) + self.logger.debug("Loaded zone: " + domain) + except DNSException: + self.logger.error("Error reading zone file:" + domain) + + def forward_dns_query(self, packet: IP): + """ + Forwards the DNS query to a real DNS resolver and returns the DNS response + """ + + dns_response = sr1( + IP(dst=self.resolver) / + UDP(sport=5000, dport=53) / + DNS(rd=1, id=packet[DNS].id, qd=packet[DNSQR]), + verbose=0 + ) + + return dns_response[DNS] + + def get_dns_query_info(self, packet: IP): + """ + Extract information from the DNS query + """ + question_name = packet[DNSQR].qname.decode("utf-8") + question_type = dns.rdatatype.to_text(packet[DNSQR].qtype) + + error = False + + # Get the first level domain name (e.g. www.google.com -> google.com) + domain_name = question_name[:-1] # Remove ending "." + try: + domain_name = get_fld(domain_name, fix_protocol=True) + except Exception as e: + self.logger.error("ERROR: Question Name: " + question_name + " - " + str(e)) + error = True + + return question_name, domain_name, question_type, error + + def get_resource_records(self, domain_name, question_name, question_type): + """ + Gets the appropriate resource record loaded earlier from the zone file + """ + resource_records = None + + data = self.zones[domain_name].get_rdataset(question_name, question_type) + + if data is None: + # NXDOMAIN + return resource_records, 0 + + # Build the resource records using scapy (DNSRR) + for record in data: + resource_record = DNSRR(rrname=question_name, type=question_type, ttl=data.ttl) + resource_record_log = "Adding record: " + question_name + ' ' + str(data.ttl) + ' ' + question_type + ' ' + + if question_type == 'MX': + resource_record_log += record.to_text() + # DNS RDATA FORMAT: Preference (16 bit integer) + Exchange (DNS Name) + resource_record[DNSRR].rdata = \ + struct.pack("!H", record.preference) + record.exchange.to_wire(None, None) + elif question_type == 'TXT': + # Retrieve only the first string in the TXT record to avoid duplicate quotes + resource_record_log += dns.rdata._escapify(record.strings[0]) + resource_record[DNSRR].rdata = dns.rdata._escapify(record.strings[0]) + else: + # Default: Records tested that work: A, NS, CNAME + resource_record_log += record.to_text() + resource_record[DNSRR].rdata = record.to_text() + + #self.logger.debug(resource_record_log) + + if resource_records is None: + resource_records = resource_record + else: + resource_records = resource_records / resource_record + + return resource_records, len(data) + + def build_dns_response(self, packet): + """ + Build the DNS response packet using one of the following methods: + 1) Load the resource record(s) from a manually configured DNS zone file (if exists) + OTHERWISE, if enabled: + 2) Send a DNS query to a DNS resolver and copy the DNS resource records + """ + + # Build response packet with empty DNS information and domain name error + dns_response = DNS(id=packet[DNS].id, rcode=3, ra=1, qr=1, qdcount=1, ancount=0, qd=packet[DNS].qd) + + # Extract information from the DNS query + question_name, domain_name, question_type, dns_query_error = self.get_dns_query_info(packet) + + info_log = "Query - Name: " + question_name + " | FLD: " + domain_name + \ + " | Record Type: " + question_type + + if domain_name in self.zones and dns_query_error is False: + # If we have a zone for this domain + self.logger.debug("Found manually configured domain: " + domain_name) + + # Get the resource records + (resource_records, count) = self.get_resource_records(domain_name, question_name, question_type) + + if count > 0: + dns_response = DNS(id=packet[DNS].id, rcode=0, ra=1, qr=1, qdcount=1, ancount=count, qd=packet[DNS].qd, + an=resource_records) + + self.logger.debug(info_log + " | Action: Zone") + + elif self.resolver is not None: + # Forward the packet to a real DNS resolver + self.logger.debug("No manually configured zone file for this domain; forwarding packet to " + self.resolver) + dns_response = self.forward_dns_query(packet) + self.logger.debug("Response from DNS resolver: " + pformat(dns_response)) + + self.logger.debug(info_log + " | Action: Forwarding") + + if self.authority is True: + dns_response[DNS].aa = 1 + + return dns_response + + def process_packet_netfilter(self, listener_packet): + """ + Callback function for each packet received by netfilter + """ + if not self.running: + return + response_packet = self.build_response_packet(listener_packet) + send(response_packet, verbose=0, iface=self.interface) + + def stop(self): + """ + Stops this server. + """ + self.running = False + + if self.listener == DNSServer.netfilter_queue: + # Give the handlers two seconds to leave the callbacks before we forcibly unbind + # the queues. + time.sleep(2) + self.nfqueue.unbind() + os.system('iptables -t mangle -D PREROUTING ' + '-p udp --dport ' + str(self.port) + ' -j NFQUEUE --queue-num %d' % self.nfqueue_num) + # Socket UDP + elif self.listener == DNSServer.socket_UDP: + if self.sock: + self.sock.close() + # Socket TCP + elif self.listener == DNSServer.socket_TCP: + if self.sock: + self.sock.close() + + ServerPlugin.stop(self) + + def build_response_packet(self, listener_packet, raw_socket=True): + """ + Build the DNS response packet + - If raw_socket is enabled include the Network and Transport Layer + """ + + packet = None + + # Netfilter + if self.listener == DNSServer.netfilter_queue: + packet = IP(listener_packet.get_payload()) + listener_packet.drop() + + # No transformations - UDP + elif self.listener == DNSServer.socket_UDP: + # Raw packet to scapy packet + packet = IP(listener_packet) + + # No transformations - TCP + elif self.listener == DNSServer.socket_TCP: + packet = DNS(listener_packet) + + if packet is None or not packet.haslayer(DNS): # if this packet does not have DNS layer + return None + + #self.logger.debug("Received the following packet " + str(self.packet_counter + 1) + ": " + pformat(packet)) + + # Ignore DNS responses + if packet[DNS].qr == 1: + #self.logger.debug("Discarding DNS response packet\n") + return None + + self.packet_counter += 1 + + # Build DNS response packet + dns_response = self.build_dns_response(packet) + + #self.logger.debug(dns_response) + + if raw_socket is True: + response_packet = IP(dst=packet[IP].src, src=packet[IP].dst) / \ + UDP(dport=packet[UDP].sport, sport=packet[UDP].dport) / \ + dns_response + + response_packet = IP(raw(response_packet)) + + else: + response_packet = dns_response + + #self.logger.debug("Response packet " + str(self.packet_counter) + ": " + + # pformat(response_packet) + "\n") + + return response_packet + + +def main(args): + """ + Run the DNS server + """ + server = DNSServer(args) + + if "dry_run" not in args: + server.start() + + return server + +if __name__ == "__main__": + main(DNSServer.get_args(sys.argv[1:])) diff --git a/plugins/dns/zones/example.com b/plugins/dns/zones/example.com new file mode 100644 index 0000000..aa3d185 --- /dev/null +++ b/plugins/dns/zones/example.com @@ -0,0 +1,26 @@ +$TTL 36000 +example.com. IN SOA ns1.example.com. hostmaster.example.com. ( + 2005081201 ; serial + 28800 ; refresh (8 hours) + 1800 ; retry (30 mins) + 2592000 ; expire (30 days) + 86400 ) ; minimum (1 day) + +example.com. 86400 NS ns1.example.com. +example.com. 86400 NS ns2.example.com. +example.com. 86400 MX 10 mail1.n2.example.com. +example.com. 86400 MX 20 mail2.example.com. +example.com. 86400 A 192.168.10.10 +example.com. 86400 A 192.168.10.11 +example.com. 86400 TXT "v=spf1 a:mail.example.com -all" + +ns1.example.com. 86400 A 192.168.1.10 +ns1.example.com. 86400 A 192.168.1.11 +ns2.example.com. 86400 A 192.168.1.20 +mail.example.com. 86400 A 192.168.2.10 +mail2.example.com. 86400 A 192.168.2.20 +www2.example.com. 86400 A 192.168.10.20 + +www.example.com. 86400 CNAME example.com. +ftp.example.com. 86400 CNAME example.com. +webmail.example.com. 86400 CNAME example.com. \ No newline at end of file diff --git a/plugins/echo/client.py b/plugins/echo/client.py new file mode 100644 index 0000000..ccda2b0 --- /dev/null +++ b/plugins/echo/client.py @@ -0,0 +1,114 @@ +""" +Client + +Run by the evaluator, echo's data back and forth to the server +""" + +import argparse +import logging +import os +import random +import socket +import sys +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import actions.utils + +from plugins.plugin_client import ClientPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class EchoClient(ClientPlugin): + """ + Defines the Echo client. + """ + name = "echo" + + def __init__(self, args): + """ + Initializes the echo client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='Echo Client') + + parser.add_argument('--injection-censor', action='store_true', help="whether this censor injects content or sends RSTs to censor") + parser.add_argument('--server', action='store', help="server to connect to") + parser.add_argument('--http-request', action='store', help="send an HTTP get request with the given hostname to the echo server") + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden GET request to the server. + """ + fitness = 0 + port = int(args["port"]) + server = args["server"] + bad_word = args["bad_word"] + msg = bad_word + if args.get("http_request"): + msg = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % args.get("http_request") + try: + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.settimeout(10) + client.connect((server, port)) + for idx in range(0, 5): + if type(msg) == str: + msg = msg.encode() + client.sendall(msg) + server_data = client.recv(1024) + logger.debug("Data recieved: %s", server_data.decode('utf-8', 'ignore')) + if server_data == msg: + fitness += 100 + elif server_data: + fitness -= 90 + break + + client.close() + # If the fitness is 0, the strategy did something to corrupt/interfere with the socket + # sending/receiving, usually by just artificially closing the connection. This behavior + # should not be rewarded with a higher fitness + if fitness == 0: + fitness -= 100 + except socket.timeout: + logger.debug("Client: Timeout") + fitness -= 100 + except socket.error as exc: + # If the censor we're running against tears down connects via RSTs, we can punish RSTs as + # if the strategy did not harm the underlying connection. However, if the censor only injects + # traffic, not resets, we should punish RSTs harshly, as the strategy likely caused it. + + if exc.errno == 104: + if args.get("injection_censor"): + fitness -= 110 + else: + fitness -= 90 + logger.debug("Client: Connection RST.") + else: + fitness -= 100 + logger.exception("Socket error caught in client echo test.") + except Exception: + logger.exception("Exception caught in client echo test.") + fitness = -120 + finally: + logger.debug("Client finished echo test.") + return fitness * 4 diff --git a/plugins/echo/server.py b/plugins/echo/server.py new file mode 100644 index 0000000..0ddd802 --- /dev/null +++ b/plugins/echo/server.py @@ -0,0 +1,95 @@ +import argparse +import os +import socket +import subprocess + +from plugins.plugin_server import ServerPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class EchoServer(ServerPlugin): + """ + Defines the Echo client. + """ + name = "echo" + def __init__(self, args): + """ + Initializes the Echo client. + """ + ServerPlugin.__init__(self) + + @staticmethod + def get_args(command): + """ + Defines arguments for this plugin + """ + super_args = ServerPlugin.get_args(command) + + parser = argparse.ArgumentParser(description='Echo Server') + + args, _ = parser.parse_known_args(command) + args = vars(args) + super_args.update(args) + return super_args + + def run(self, args, logger): + """ + Initializes the Echo server. + """ + logger.debug("Echo server initializing") + try: + port = int(args["port"]) + control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Allow socket re-use + control_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_address = ('0.0.0.0', port) + logger.debug("Binding to server address 0.0.0.0:%d" % port) + control_socket.bind(server_address) + control_socket.settimeout(5) + control_socket.listen(1) + except: + logger.exception("Caught exception in echo run") + return + + try: + connection, client_address = self.get_request(control_socket) + if not connection: + logger.error("Failed to get connection") + return + # Echo data back and forth + for i in range(0, 5): + data = connection.recv(1024) + connection.sendall(data) + + connection.close() + except socket.error as e: + if e.errno == 104: + logger.debug("Server: Connection RST.") + else: + logger.debug("Server: Client quit.") + except socket.ConnectionResetError: + logger.debug("Server: Connection RST.") + except Exception: + logger.exception("Failed during server communication.") + finally: + logger.debug("Server exiting") + + def get_request(self, control_socket): + """ + Get a request from the socket. + """ + while True: + try: + sock, addr = control_socket.accept() + sock.settimeout(5) + return (sock, addr) + except socket.timeout: + pass + return (None, None) + + def stop(self): + """ + Stops this server. + """ + ServerPlugin.stop(self) diff --git a/plugins/http/client.py b/plugins/http/client.py new file mode 100644 index 0000000..b6b68ff --- /dev/null +++ b/plugins/http/client.py @@ -0,0 +1,101 @@ +""" +Run by the evaluator, tries to make a GET request to a given server +""" + +import argparse +import logging +import os +import random +import socket +import sys +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import external_sites +import actions.utils + +from plugins.plugin_client import ClientPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class HTTPClient(ClientPlugin): + """ + Defines the HTTP client. + """ + name = "http" + + def __init__(self, args): + """ + Initializes the HTTP client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='HTTP Client', prog="http/client.py") + + parser.add_argument('--host-header', action='store', default="", help='specifies host header for HTTP request') + parser.add_argument('--injected-http-contains', action='store', default="", help='checks if injected http response contains string') + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden GET request to the server. + """ + fitness = 0 + url = args.get("server", "") + assert url, "Cannot launch HTTP test with no server" + if not url.startswith("http://"): + url = "http://" + url + headers = {} + if args.get('host_header'): + headers["Host"] = args.get('host_header') + + # If we've been given a non-standard port, append that to the URL + port = args.get("port", 80) + if port != 80: + url += ":%s" % str(port) + + url += "?q=%s" % args.get("bad_word") + + injected_http = args.get("injected_http_contains") + try: + res = requests.get(url, allow_redirects=False, timeout=3, headers=headers) + logger.debug(res.text) + # If we need to monitor for an injected response, check that here + if injected_http and injected_http in res.text: + fitness -= 90 + else: + fitness += 100 + except (requests.exceptions.ConnectionError, ConnectionResetError) as exc: + logger.exception("Connection RST.") + fitness -= 90 + except urllib.error.URLError as exc: + logger.debug(exc) + fitness += -101 + # Timeouts generally mean the strategy killed the TCP stream. + # HTTPError usually mean the request was destroyed. + # Punish this more harshly than getting caught by the censor. + except (requests.exceptions.Timeout, requests.exceptions.HTTPError) as exc: + logger.debug(exc) + fitness += -120 + except Exception: + logger.exception("Exception caught in HTTP test to site %s.", url) + fitness += -100 + return fitness * 4 diff --git a/plugins/http/external_sites.py b/plugins/http/external_sites.py new file mode 100755 index 0000000..b5fb2de --- /dev/null +++ b/plugins/http/external_sites.py @@ -0,0 +1,3 @@ +DNS_BLOCKED = ["google.com", "facebook.com"] +EXTERNAL_SITES = ['http://www.wikipedia.org', 'http://www.yahoo.com', 'http://www.amazon.com', 'http://www.live.com', 'http://www.netflix.com', 'http://www.yandex.ru', 'http://www.yahoo.co.jp', 'http://www.vk.com', 'http://www.mail.ru', 'http://www.ebay.com', 'http://www.bing.com', 'http://www.imdb.com', 'http://www.linkedin.com', 'http://www.microsoft.com', 'http://www.livejasmin.com', 'http://www.office.com', 'http://www.naver.com', 'http://www.amazon.co.jp', 'http://www.imgur.com', 'http://www.apple.com', 'http://www.espn.com', 'http://www.wikia.com', 'http://www.popads.net', 'http://www.paypal.com', 'http://www.msn.com', 'http://www.stackoverflow.com', 'http://www.github.com', 'http://www.amazon.in', 'http://www.amazon.de', 'http://www.cobalten.com', 'http://www.amazon.co.uk', 'http://www.f5v1x3kgv5.com', 'http://www.adobe.com', 'http://www.thestartmagazine.com', 'http://www.roblox.com', 'http://www.gearbest.com', 'http://www.booking.com', 'http://www.aparat.com', 'http://www.rakuten.co.jp', 'http://www.bukalapak.com', 'http://www.cnn.com', 'http://www.softonic.com', 'http://www.craigslist.org', 'http://www.stackexchange.com', 'http://www.swiftviz.net', 'http://www.nicovideo.jp', 'http://www.i62e2b4mfy.com', 'http://www.openload.co', 'http://www.speakol.com', 'http://www.onlinesbi.com', 'http://www.spotify.com', 'http://www.nih.gov', 'http://www.avito.ru', 'http://www.ebay.de', 'http://www.mediafire.com', 'http://www.bodelen.com', 'http://www.amazonaws.com', 'http://www.theguardian.com', 'http://www.detik.com', 'http://www.nextoptim.com', 'http://www.hotstar.com', 'http://www.walmart.com', 'http://www.spouscontentdelivery.info', 'http://www.chase.com', 'http://www.cnet.com', 'http://www.etsy.com', 'http://www.researchgate.net', 'http://www.amousinded.info', 'http://www.globo.com', 'http://www.flipkart.com', 'http://www.otvfoco.com.br', 'http://www.deviantart.com', 'http://www.digitaldsp.com', 'http://www.amazon.fr', 'http://www.gamepedia.com', 'http://www.yts.am', 'http://www.daum.net', 'http://www.bet9ja.com', 'http://www.bestbuy.com', 'http://www.indeed.com', 'http://www.uol.com.br', 'http://www.instructure.com', 'http://www.cricbuzz.com', 'http://www.hulu.com', 'http://www.wikihow.com', 'http://www.doublepimpssl.com', 'http://www.messenger.com', 'http://www.amazon.it', 'http://www.w3schools.com', 'http://www.vice.com', 'http://www.tokopedia.com', 'http://www.mercadolivre.com.br', 'http://www.godaddy.com', 'http://www.sciencedirect.com', 'http://www.jf71qh5v14.com', 'http://www.onlinevideoconverter.com', 'http://www.mozilla.org', 'http://www.1337x.to', 'http://www.washingtonpost.com', 'http://www.kinopoisk.ru', 'http://www.dailymail.co.uk', 'http://www.ikea.com', 'http://www.liputan6.com', 'http://www.steampowered.com', 'http://www.yelp.com', 'http://www.naganoadigei.com', 'http://www.ebay-kleinanzeigen.de', 'http://www.rambler.ru', 'http://www.livejournal.com', 'http://www.genius.com', 'http://www.rutracker.org', 'http://www.savefrom.net', 'http://www.digikala.com', 'http://www.op.gg', 'http://www.speedtest.net', 'http://www.amazon.es', 'http://www.primevideo.com', 'http://www.doublepimp.com', 'http://www.quizlet.com', 'http://www.zillow.com', 'http://www.youm7.com', 'http://www.fandom.com', 'http://www.nfl.com', 'http://www.indiatimes.com', 'http://www.kissanime.ru', 'http://www.amazon.ca', 'http://www.kompas.com', 'http://www.target.com', 'http://www.rarbg.to', 'http://www.asos.com', 'http://www.gamespot.com', 'http://www.xfinity.com', 'http://www.siteadvisor.com', 'http://www.browsergames2018.com', 'http://www.news-speaker.com', 'http://www.varzesh3.com', 'http://www.canva.com', 'http://www.wikimedia.org', 'http://www.battle.net', 'http://www.ladbible.com', 'http://www.tripadvisor.com', 'http://www.guildofangels.net', 'http://www.popcash.net', 'http://www.reverso.net', 'http://www.abs-cbn.com', 'http://www.pinimg.com', 'http://www.weather.com', 'http://www.foxnews.com', 'http://www.souq.com', 'http://www.irctc.co.in', 'http://www.livescore.com', 'http://www.kakaku.com', 'http://www.tvbs.com.tw', 'http://www.buzzfeed.com', 'http://www.academia.edu', 'http://www.rt.com', 'http://www.9gag.com', 'http://www.livedoor.com', 'http://www.ign.com', 'http://www.crptentry.com', 'http://www.allegro.pl', 'http://www.bankofamerica.com', 'http://www.freepik.com', 'http://www.blackboard.com', 'http://www.wellsfargo.com', 'http://www.salesforce.com', 'http://www.udemy.com', 'http://www.patreon.com', 'http://www.ouo.io', 'http://www.rottentomatoes.com', 'http://www.forbes.com', 'http://www.kooora.com', 'http://www.gfycat.com', 'http://www.olx.ua', 'http://www.myshopify.com', 'http://www.airbnb.com', 'http://www.espncricinfo.com', 'http://www.aol.com', 'http://www.namu.wiki', 'http://www.leagueoflegends.com', 'http://www.ameblo.jp', 'http://www.tistory.com', 'http://www.exoclick.com', 'http://www.wetransfer.com', 'http://www.hellomagazine.com', 'http://www.samsung.com', 'http://www.namnak.com', 'http://www.myway.com', 'http://www.orange.fr', 'http://www.wordreference.com', 'http://www.leboncoin.fr', 'http://www.marca.com', 'http://www.userapi.com', 'http://www.wowhead.com', 'http://www.slickdeals.net', 'http://www.force.com', 'http://www.trello.com', 'http://www.weebly.com', 'http://www.taboola.com', 'http://www.fiverr.com', 'http://www.factaholics.com', 'http://www.seasonvar.ru', 'http://www.thesaurus.com', 'http://www.cambridge.org', 'http://www.dmm.co.jp', 'http://www.sourceforge.net', 'http://www.yourporn.sexy', 'http://www.ebc.net.tw', 'http://www.chegg.com', 'http://www.infourok.ru', 'http://www.huffingtonpost.com', 'http://www.free.fr', 'http://www.torrentz2.eu', 'http://www.pikabu.ru', 'http://www.dcinside.com', 'http://www.goodreads.com', 'http://www.rutube.ru', 'http://www.okdiario.com', 'http://www.ultimate-guitar.com', 'http://www.subscene.com', 'http://www.sberbank.ru', 'http://www.zimuzu.tv', 'http://www.indoxxi.vip', 'http://www.myanimelist.net', 'http://www.vacaneedasap.com', 'http://www.ria.ru', 'http://www.nike.com', 'http://www.grid.id', 'http://www.outbrain.com', 'http://www.elmogaz.com', 'http://www.kaskus.co.id', 'http://www.nba.com', 'http://www.premierleague.com', 'http://www.wish.com', 'http://www.discogs.com', 'http://www.hp.com', 'http://www.buy123.com.tw', 'http://www.divar.ir', 'http://www.prezi.com', 'http://www.znanija.com', 'http://www.playstation.com', 'http://www.americanexpress.com', 'http://www.hola.com', 'http://www.idntimes.com', 'http://www.accuweather.com', 'http://www.wordpress.org', 'http://www.wp.pl', 'http://www.nexusmods.com', 'http://www.tamilrockers.by', 'http://www.wiktionary.org', 'http://www.onet.pl', 'http://www.daraz.pk', 'http://www.mawdoo3.com', 'http://www.crunchyroll.com', 'http://www.cinecalidad.to', 'http://www.motorsport.com', 'http://www.urbandictionary.com', 'http://www.coinmarketcap.com', 'http://www.capitalone.com', 'http://www.shaparak.ir', 'http://www.albawabhnews.com', 'http://www.rapidvideo.com', 'http://www.ndtv.com', 'http://www.ouedkniss.com', 'http://www.upgers-armine.com', 'http://www.fb.ru', 'http://www.usatoday.com', 'http://www.office365.com', 'http://www.yandex.kz', 'http://www.healthline.com', 'http://www.prothomalo.com', 'http://www.olx.pl', 'http://www.hm.com', 'http://www.ensonhaber.com', 'http://www.uniqlo.com', 'http://www.oath.com', 'http://www.mercadolibre.com.mx', 'http://www.ebates.com', 'http://www.animeyt.tv', 'http://www.mercadolibre.com.ar', 'http://www.ebay.it', 'http://www.asus.com', 'http://www.nyaa.si', 'http://www.evernote.com', 'http://www.flvto.biz', 'http://www.telewebion.com', 'http://www.wix.com', 'http://www.banggood.com', 'http://www.cbssports.com', 'http://www.go.com', 'http://www.oload.fun', 'http://www.springer.com', 'http://www.slack.com', 'http://www.lazada.com.my', 'http://www.trustednotice.news', 'http://www.sindonews.com', 'http://www.pantip.com', 'http://www.utorrent.com', 'http://www.pop.bid', 'http://www.drom.ru', 'http://www.pixabay.com', 'http://www.glassdoor.com', 'http://www.gmx.net', 'http://www.tutorialspoint.com', 'http://www.hdfcbank.com', 'http://www.wiley.com', 'http://www.newegg.com', 'http://www.upwork.com', 'http://www.chouftv.ma', 'http://www.animeflv.net', 'http://www.caliente.mx', 'http://www.zendesk.com', 'http://www.egy.best', 'http://www.spotscenered.info', 'http://www.mit.edu', 'http://www.ebay.com.au', 'http://www.geeksforgeeks.org', 'http://www.4chan.org', 'http://www.goo.ne.jp', 'http://www.convert2mp3.net', 'http://www.nordstrom.com', 'http://www.kumparan.com', 'http://www.bandcamp.com', 'http://www.kohls.com', 'http://www.yadi.sk', 'http://www.usps.com', 'http://www.costco.com', 'http://www.lenta.ru', 'http://www.icloud.com', 'http://www.notification-time.com', 'http://www.webmd.com', 'http://www.westernjournal.com', 'http://www.skype.com', 'http://www.voot.com', 'http://www.flirt4free.com', 'http://www.gamer.com.tw', 'http://www.web.de', 'http://www.epicgames.com', 'http://www.dell.com', 'http://www.shein.com', 'http://www.livetv.sx', 'http://www.videodownloadconverter.com', 'http://www.independent.co.uk', 'http://www.keyrolan.com', 'http://www.xda-developers.com', 'http://www.spiegel.de', 'http://www.hespress.com', 'http://www.mobafire.com', 'http://www.drive2.ru', 'http://www.goal.com', 'http://www.elsevier.com', 'http://www.gosuslugi.ru', 'http://www.shopify.com', 'http://www.gmarket.co.kr', 'http://www.citi.com', 'http://www.as.com', 'http://www.elfagr.com', 'http://www.mailchimp.com', 'http://www.gsmarena.com', 'http://www.uod2quk646.com', 'http://www.advertserve.com', 'http://www.shopee.tw', 'http://www.wayfair.com', 'http://www.wildberries.ru', 'http://www.sonyliv.com', 'http://www.paytm.com', 'http://www.repubblica.it', 'http://www.themeforest.net', 'http://www.beytoote.com', 'http://www.allrecipes.com', 'http://www.oracle.com', 'http://www.grammarly.com', 'http://www.khanacademy.org', 'http://www.chess.com', 'http://www.digialm.com', 'http://www.drudgereport.com', 'http://www.freejobalert.com', 'http://www.vseigru.net', 'http://www.intuit.com', 'http://www.vcommission.com', 'http://www.kickstarter.com', 'http://www.thefreedictionary.com', 'http://www.manoramaonline.com', 'http://www.gyazo.com', 'http://www.merriam-webster.com', 'http://www.getlnk3.com', 'http://www.billdesk.com', 'http://www.streamable.com', 'http://www.vidio.com', 'http://www.archiveofourown.org', 'http://www.alwafd.news', 'http://www.trendyol.com', 'http://www.biobiochile.cl', 'http://www.filehippo.com', 'http://www.express.co.uk', 'http://www.flashscore.com', 'http://www.diply.com', 'http://www.kijiji.ca', 'http://www.newstarads.com', 'http://www.bild.de', 'http://www.sabq.org', 'http://www.torrent9.ph', 'http://www.kizlarsoruyor.com', 'http://www.artstation.com', 'http://www.patch.com', 'http://www.icicibank.com', 'http://www.y8.com', 'http://www.cnbc.com', 'http://www.att.com', 'http://www.rarbgprx.org', 'http://www.taleo.net', 'http://www.yaplakal.com', 'http://www.telegraph.co.uk', 'http://www.alnaharegypt.com', 'http://www.5ch.net', 'http://www.kapanlagi.com', 'http://www.rapidgator.net', 'http://www.movierulz.ht', 'http://www.cdiscount.com', 'http://www.techradar.com', 'http://www.kinokrad.co', 'http://www.velocecdn.com', 'http://www.inven.co.kr', 'http://www.thewhizmarketing.com', 'http://www.rotumal.com', 'http://www.theverge.com', 'http://www.t-online.de', 'http://www.unblocked.gdn', 'http://www.weblio.jp', 'http://www.investopedia.com', 'http://www.uzone.id', 'http://www.pornq.com', 'http://www.ozon.ru', 'http://www.2m.ma', 'http://www.ecosia.org', 'http://www.4pda.ru', 'http://www.movie2free.com', 'http://www.libero.it', 'http://www.ruliweb.com', 'http://www.hepsiburada.com', 'http://www.giphy.com', 'http://www.yandex.com.tr', 'http://www.taimienphi.vn', 'http://www.bitly.com', 'http://www.mercari.com', 'http://www.okta.com', 'http://www.cryptobrowser.site', 'http://www.ivi.ru', 'http://www.myanmarload.com', 'http://www.rutor.info', 'http://www.nur.kz', 'http://www.chip.de', 'http://www.expedia.com', 'http://www.dictionary.com', 'http://www.discover.com', 'http://www.zoho.com', 'http://www.metropoles.com', 'http://www.merdeka.com', 'http://www.buffstreamz.com', 'http://www.groupon.com', 'http://www.vesti.ru', 'http://www.lowes.com', 'http://www.bhphotovideo.com', 'http://www.uptobox.com', 'http://www.uploaded.net', 'http://www.so3ody.com', 'http://www.unsplash.com', 'http://www.elmundo.es', 'http://www.freeadult.games', 'http://www.olx.com.br', 'http://www.ieee.org', 'http://www.wunderground.com', 'http://www.ninisite.com', 'http://www.politico.com', 'http://www.sputniknews.com', 'http://www.ea.com', 'http://www.mgid.com', 'http://www.totalsportek.com', 'http://www.lazada.com.ph', 'http://www.ebay.fr', 'http://www.rediff.com', 'http://www.gizmodo.com', 'http://www.y2mate.com', 'http://www.taringa.net', 'http://www.mobile.de', 'http://www.s4yxaqyq95.com', 'http://www.thethao247.vn', 'http://www.eztv.re', 'http://www.prom.ua', 'http://www.interia.pl', 'http://www.kissasian.sh', 'http://www.hubspot.com', 'http://www.smallpdf.com', 'http://www.wondershare.com', 'http://www.feebee.com.tw', 'http://www.liquipedia.net', 'http://www.norton.com', 'http://www.skysports.com', 'http://www.gazeta.ru', 'http://www.npr.org', 'http://www.cnnindonesia.com', 'http://www.tomshardware.com', 'http://www.commank.pro', 'http://www.tandfonline.com', 'http://www.nsportal.ru', 'http://www.wargaming.net', 'http://www.cookpad.com', 'http://www.ilovepdf.com', 'http://www.rozetka.com.ua', 'http://www.agoda.com', 'http://www.lazada.co.id', 'http://www.yifysubtitles.com', 'http://www.engadget.com', 'http://www.zing.vn', 'http://www.addroplet.com', 'http://www.fishki.net', 'http://www.coursera.org', 'http://www.mercadolibre.com.ve', 'http://www.alodokter.com', 'http://www.americanas.com.br', 'http://www.pexels.com', 'http://www.umblr.com', 'http://www.auction.co.kr', 'http://www.gmanetwork.com', 'http://www.okcupid.com', 'http://www.tradingview.com', 'http://www.elbalad.news', 'http://www.verizonwireless.com', 'http://www.zaycev.net', 'http://www.nbcnews.com', 'http://www.lefigaro.fr', 'http://www.squarespace.com', 'http://www.latimes.com', 'http://www.fanfiction.net', 'http://www.letyshops.com', 'http://www.shopee.co.id', 'http://www.subito.it', 'http://www.hatena.ne.jp', 'http://www.filmitorrent.org', 'http://www.trustpilot.com', 'http://www.redwap.me', 'http://www.limetorrents.info', 'http://www.vlive.tv', 'http://www.study.com', 'http://www.yandex.by', 'http://www.blizzard.com', 'http://www.ytmp3.cc', 'http://www.lifewire.com', 'http://www.dmm.com', 'http://www.heavy.com', 'http://www.tebyan.net', 'http://www.cimaclub.com', 'http://www.kp.ru', 'http://www.11st.co.kr', 'http://www.exhentai.org', 'http://www.givemesport.com', 'http://www.zara.com', 'http://www.cdnondemand.org', 'http://www.avast.com', 'http://www.jstor.org', 'http://www.sportbible.com', 'http://www.blog.me', 'http://www.hotels.com', 'http://www.fromdoctopdf.com', 'http://www.calcch.com', 'http://www.namasha.com', 'http://www.championat.com', 'http://www.dailyfeed.co.uk', 'http://www.celeritascdn.com', 'http://www.naukri.com', 'http://www.brilio.net', 'http://www.biblegateway.com', 'http://www.pcgamer.com', 'http://www.bancodevenezuela.com', 'http://www.sports.ru', 'http://www.autodesk.com', 'http://www.shopee.com.my', 'http://www.reundcwkqvctq.com', 'http://www.hotnewhiphop.com', 'http://www.xbox.com', 'http://www.jw.org', 'http://www.mileroticos.com', 'http://www.onlinefilm-hd.com', 'http://www.linkshrink.net', 'http://www.sporx.com', 'http://www.jamnews.com', 'http://www.nbcsports.com', 'http://www.nvidia.com', 'http://www.ficbook.net', 'http://www.sephora.com', 'http://www.list-manage.com', 'http://www.bankmellat.ir', 'http://www.inquirer.net', 'http://www.bleacherreport.com', 'http://www.feedly.com', 'http://www.humblebundle.com', 'http://www.monografias.com', 'http://www.makemytrip.com', 'http://www.mathrubhumi.com', 'http://www.mathworks.com', 'http://www.getpocket.com', 'http://www.directv.com', 'http://www.gogoanimes.co', 'http://www.otto.de', 'http://www.adme.ru', 'http://www.naver.jp', 'http://www.tarafdari.com', 'http://www.afreecatv.com', 'http://www.wp.com', 'http://www.nhl.com', 'http://www.ancestry.com', 'http://www.box.com', 'http://www.gap.com', 'http://www.appvalley.vip', 'http://www.playground.ru', 'http://www.stanford.edu', 'http://www.rus.ec', 'http://www.lenovo.com', 'http://www.ups.com', 'http://www.elwatannews.com', 'http://www.intel.com', 'http://www.digitaltrends.com', 'http://www.neostreamz.com', 'http://www.pirateproxy.gdn', 'http://www.addthis.com', 'http://www.allocine.fr', 'http://www.eskimi.com', 'http://www.bles.com', 'http://www.streamango.com', 'http://www.tomsguide.com', 'http://www.academic.ru', 'http://www.foodnetwork.com', 'http://www.memurlar.net', 'http://www.gidonline.in', 'http://www.rbc.ru', 'http://www.mheducation.com', 'http://www.getlnk1.com', 'http://www.retailmenot.com', 'http://www.realtor.com', 'http://www.wikiwand.com', 'http://www.europa.eu', 'http://www.neobux.com', 'http://www.tvplusnewtab.com', 'http://www.auto.ru', 'http://www.iherb.com', 'http://www.aminoapps.com', 'http://www.olymptrade.com', 'http://www.investing.com', 'http://www.bestadbid.com', 'http://www.ccm.net', 'http://www.torrentwal.net', 'http://www.hamariweb.com', 'http://www.dytt8.net', 'http://www.cda.pl', 'http://www.kinogo.cc', 'http://www.1tv.ru', 'http://www.ukr.net', 'http://www.delta.com', 'http://www.xhamsterlive.com', 'http://www.pandora.com', 'http://www.fullhdfilmizlesene.net', 'http://www.instructables.com', 'http://www.dns-shop.ru', 'http://www.mayoclinic.org', 'http://www.knowyourmeme.com', 'http://www.yallakora.com', 'http://www.duolingo.com', 'http://www.vnexpress.net', 'http://www.worldstarhiphop.com', 'http://www.corriere.it', 'http://www.animevost.org', 'http://www.constintptr.com', 'http://www.fehobmasr.com', 'http://www.timeanddate.com', 'http://www.tabelog.com', 'http://www.getlnk2.com', 'http://www.nature.com', 'http://www.ccleaner.com', 'http://www.termometropolitico.it', 'http://www.futbin.com', 'http://www.beinsports.com', 'http://www.cbsnews.com', 'http://www.blastingnews.com', 'http://www.thingiverse.com', 'http://www.okezone.com', 'http://www.harvard.edu', 'http://www.asahi.com', 'http://www.ask.fm', 'http://www.surveymonkey.com', 'http://www.elitetorrent.biz', 'http://www.tnt-online.ru', 'http://www.104.com.tw', 'http://www.telegraf.com.ua', 'http://www.1fichier.com', 'http://www.tfetimes.com', 'http://www.thepiratebay.rocks', 'http://www.imagetwist.com', 'http://www.marriott.com', 'http://www.meetup.com', 'http://www.dominos.com', 'http://www.dafont.com', 'http://www.indiamart.com', 'http://www.urdupoint.com', 'http://www.b9good.com', 'http://www.kayak.com', 'http://www.wasabisyrup.com', 'http://www.chron.com', 'http://www.marumaru.in', 'http://www.myntra.com', 'http://www.performanceonclick.com', 'http://www.dstv.com', 'http://www.eventbrite.com', 'http://www.torrentmap.com', 'http://www.arxiv.org', 'http://www.geniusdexchange.com', 'http://www.livesport.ws', 'http://www.united.com', 'http://www.lequipe.fr', 'http://www.fedex.com', 'http://www.news.com.au', 'http://www.theatlantic.com', 'http://www.forgeofempires.com', 'http://www.warframe.com', 'http://www.teacherspayteachers.com', 'http://www.hltv.org', 'http://www.1xpwyy.host', 'http://www.ibps.in', 'http://www.fatosdesconhecidos.com.br', 'http://www.himado.in', 'http://www.zougla.gr', 'http://www.lds.org', 'http://www.echo.msk.ru', 'http://www.tinder.com', 'http://www.runtnc.net', 'http://www.zomato.com', 'http://www.mvideo.ru', 'http://www.envato.com', 'http://www.fmovies.to', 'http://www.donga.com', 'http://www.online-convert.com', 'http://www.hentaihaven.org', 'http://www.socialnewpages.com', 'http://www.paytmmall.com', 'http://www.bolshoyvopros.ru', 'http://www.nasa.gov', 'http://www.britannica.com', 'http://www.bluestacks.com', 'http://www.aksam.com.tr', 'http://www.asriran.com', 'http://www.internetdownloadmanager.com', 'http://www.redfin.com', 'http://www.kisspng.com', 'http://www.dnevnik.ru', 'http://www.indianexpress.com', 'http://www.paytm.in', 'http://www.spectrum.net', 'http://www.mangakakalot.com', 'http://www.furaffinity.net', 'http://www.liveinternet.ru', 'http://www.creditkarma.com', 'http://www.snapdeal.com', 'http://www.mptentry.com', 'http://www.codepen.io', 'http://www.ebay.es', 'http://www.southwest.com', 'http://www.howtogeek.com', 'http://www.argaam.com', 'http://www.dailycaller.com', 'http://www.ted.com', 'http://www.nypost.com', 'http://www.cloudflare.com', 'http://www.ppy.sh', 'http://www.fidelity.com', 'http://www.bungie.net', 'http://www.jimdo.com', 'http://www.ronaldo7.net', 'http://www.sfr.fr', 'http://www.usnews.com', 'http://www.infobae.com', 'http://www.gaana.com', 'http://www.aa.com', 'http://www.setare.com', 'http://www.hqq.tv', 'http://www.bol.uol.com.br', 'http://www.askubuntu.com', 'http://www.worldofwarcraft.com', 'http://www.lentainform.com', 'http://www.curseforge.com', 'http://www.yalla-shoot.com', 'http://www.ouo.press', 'http://www.gnula.nu', 'http://www.youla.ru', 'http://www.jugantor.com', 'http://www.torrenthaja.com', 'http://www.asana.com', 'http://www.jeuxvideo.com', 'http://www.itorrents.org', 'http://www.letras.mus.br', 'http://www.pipeschannels.com', 'http://www.superuser.com', 'http://www.33sk.tv', 'http://www.bookmyshow.com', 'http://www.elcomercio.pe', 'http://www.strava.com', 'http://www.sapo.pt', 'http://www.sinoptik.ua', 'http://www.urbanoutfitters.com', 'http://www.psu.edu', 'http://www.clarin.com', 'http://www.yandex.com', 'http://www.probuilds.net', 'http://www.xe.com', 'http://www.g2a.com', 'http://www.zaful.com', 'http://www.fotostrana.ru', 'http://www.dmhy.org', 'http://www.yjc.ir', 'http://www.jio.com', 'http://www.sportbox.ru', 'http://www.igg-games.com', 'http://www.internetspeedtracker.com', 'http://www.thoughtco.com', 'http://www.swatchseries.to', 'http://www.www.gov.uk', 'http://www.commentcamarche.net', 'http://www.lifo.gr', 'http://www.prnt.sc', 'http://www.lostfilm.tv', 'http://www.last.fm', 'http://www.fanserials.media', 'http://www.abc.es', 'http://www.bt.com', 'http://www.xvideosex.site', 'http://www.ibm.com', 'http://www.python.org', 'http://www.nutaku.net', 'http://www.getawesome1.com', 'http://www.readms.net', 'http://www.francetvinfo.fr', 'http://www.thehill.com', 'http://www.sonyentertainmentnetwork.com', 'http://www.vkuseraudio.net', 'http://www.nairaland.com', 'http://www.hh.ru', 'http://www.farsnews.com', 'http://www.libreape.com', 'http://www.gazzetta.it', 'http://www.href.li', 'http://www.otzovik.com', 'http://www.pngtree.com', 'http://www.nintendo.com', 'http://www.manyvids.com', 'http://www.fivethirtyeight.com', 'http://www.ipleer.fm', 'http://www.bayt.com', 'http://www.tver.jp', 'http://www.bamilo.com', 'http://www.labanquepostale.fr', 'http://www.lifehacker.ru', 'http://www.idnes.cz', 'http://www.blogfa.com', 'http://www.makeuseof.com', 'http://www.jumia.com.ng', 'http://www.avg.com', 'http://www.wease.im', 'http://www.avito.ma', 'http://www.adp.com', 'http://www.proxyonetpb.pet', 'http://www.getintopc.com', 'http://www.cima4u.tv', 'http://www.pulzo.com', 'http://www.ameba.jp', 'http://www.idealo.de', 'http://www.dazn.com', 'http://www.chinatimes.com', 'http://www.over-blog.com', 'http://www.welt.de', 'http://www.avaz.ba', 'http://www.ryanair.com', 'http://www.kissmanga.com', 'http://www.shopee.co.th', 'http://www.bola.net', 'http://www.zappos.com', 'http://www.horriblesubs.info', 'http://www.reverb.com', 'http://www.secureserver.net', 'http://www.livetvcdn.net', 'http://www.sofascore.com', 'http://www.msi.com', 'http://www.the123movies.org', 'http://www.gdz.ru', 'http://www.getawesome2.com', 'http://www.getawesome3.com', 'http://www.kknews.cc', 'http://www.celebrityinsider.org', 'http://www.skyscanner.net', 'http://www.p30download.com', 'http://www.mynet.com', 'http://www.getawesome4.com', 'http://www.yournewtab.com', 'http://www.newstrend.news', 'http://www.nextdoor.com', 'http://www.ssl2anyone3.com', 'http://www.misdem.pro', 'http://www.ecollege.com', 'http://www.ubi.com', 'http://www.lazada.vn', 'http://www.focus.de', 'http://www.aktuality.sk', 'http://www.audible.com', 'http://www.cb01.news', 'http://www.getawesome6.com', 'http://www.rakuten.com', 'http://www.viki.com', 'http://www.getawesome5.com', 'http://www.viva.co.id', 'http://www.amazon.com.mx', 'http://www.abola.pt', 'http://www.nexon.com', 'http://www.thesun.co.uk', 'http://www.mmo-champion.com', 'http://www.webtoons.com', 'http://www.yodobashi.com', 'http://www.pole-emploi.fr', 'http://www.gumtree.com.au', 'http://www.purdue.edu', 'http://www.citilink.ru', 'http://www.shop-apotheke.com', 'http://www.123movie.cc', 'http://www.shadbase.com', 'http://www.indianrail.gov.in', 'http://www.cengage.com', 'http://www.2conv.com', 'http://www.eroterest.net', 'http://www.biqle.ru', 'http://www.isna.ir', 'http://www.ed.gov', 'http://www.jd.id', 'http://www.otomoto.pl', 'http://www.azlyrics.com', 'http://www.people.com', 'http://www.streamlabs.com', 'http://www.neoldu.com', 'http://www.adultmmogame.com', 'http://www.arcot.com', 'http://www.etoland.co.kr', 'http://www.everydayhealth.com.tw', 'http://www.filecrypt.cc', 'http://www.shutterfly.com', 'http://www.lun.com', 'http://www.moe.gov.sa', 'http://www.ucoz.ru', 'http://www.mk.ru', 'http://www.weather.gov', 'http://www.kicker.de', 'http://www.valuecommerce.com', 'http://www.usaa.com', 'http://www.tamasha.com', 'http://www.psychologytoday.com', 'http://www.sme.sk', 'http://www.uptimecdn.com', 'http://www.mangarock.com', 'http://www.torrent-games.net', 'http://www.tut.by', 'http://www.sporttube.com', 'http://www.codecanyon.net', 'http://www.file-upload.com', 'http://www.coupang.com', 'http://www.redbubble.com', 'http://www.getawesome9.com', 'http://www.uber.com', 'http://www.cbs.com', 'http://www.programme-tv.net', 'http://www.multiurok.ru', 'http://www.audienceline.com', 'http://www.blibli.com', 'http://www.gazetaexpress.com', 'http://www.bs.to', 'http://www.thepiratebay3.org', 'http://www.dawn.com', 'http://www.airedesantafe.com.ar', 'http://www.java.com', 'http://www.worldoftanks.ru', 'http://www.sdamgia.ru', 'http://www.fitgirl-repacks.site', 'http://www.pornhubpremium.com', 'http://www.mp3teca.com', 'http://www.channelmyanmar.org', 'http://www.oceanofgames.com', 'http://www.overleaf.com', 'http://www.vanguardngr.com', 'http://www.bustle.com', 'http://www.abril.com.br', 'http://www.skroutz.gr', 'http://www.vox.com', 'http://www.lazada.sg', 'http://www.jcpenney.com', 'http://www.wuxiaworld.com', 'http://www.sky.com', 'http://www.javfor.me', 'http://www.shopee.vn', 'http://www.dunia21.net', 'http://www.tempo.co', 'http://www.cargurus.com', 'http://www.filmaffinity.com', 'http://www.nordstromrack.com', 'http://www.1xvgsc.host', 'http://www.fast.com', 'http://www.247sports.com', 'http://www.libgen.io', 'http://www.ikvagxovc.com', 'http://www.acs.org', 'http://www.mediaset.it', 'http://www.abcnews.go.com', 'http://www.impress.co.jp', 'http://www.clien.net', 'http://www.litres.ru', 'http://www.justdial.com', 'http://www.oup.com', 'http://www.easypdfcombine.com', 'http://www.metacritic.com', 'http://www.collinsdictionary.com', 'http://www.jd.hk', 'http://www.pcpartpicker.com', 'http://www.katcr.co', 'http://www.myscore.com.ua', 'http://www.paheal.net', 'http://www.champion.gg', 'http://www.minecraft.net', 'http://www.2ch.net', 'http://www.thomann.de', 'http://www.yeniakit.com.tr', 'http://www.kino-teatr.ru', 'http://www.mail.com', 'http://www.watchcartoononline.com', 'http://www.vkuservideo.net', 'http://www.ygdy8.com', 'http://www.sagepub.com', 'http://www.ebay.ca', 'http://www.gog.com', 'http://www.tagged.com', 'http://www.tmz.com', 'http://www.tamilmv.biz', 'http://www.viu.com', 'http://www.dagospia.com', 'http://www.zooqle.com', 'http://www.mirror.co.uk', 'http://www.liontravel.com', 'http://www.yout.com', 'http://www.porno365.xxx', 'http://www.gonews.pro', 'http://www.velocitycdn.com', 'http://www.alsbbora.info', 'http://www.flaticon.com', 'http://www.appdatum.com', 'http://www.moegirl.org', 'http://www.fandango.com', 'http://www.canalrcn.com', 'http://www.pornolab.net', 'http://www.gumtree.com', 'http://www.irecommend.ru', 'http://www.trulia.com', 'http://www.tasnimnews.com', 'http://www.gamestop.com', 'http://www.ripple.is', 'http://www.citationmachine.net', 'http://www.kissanime.ac', 'http://www.edx.org', 'http://www.milanuncios.com', 'http://www.film2movie.us', 'http://www.webofknowledge.com', 'http://www.lapatilla.com', 'http://www.opensubtitles.org', 'http://www.1873083787.rsc.cdn77.org', 'http://www.k2s.cc', 'http://www.pixhost.to', 'http://www.akoam.net', 'http://www.heydouga.com', 'http://www.mymovies.it', 'http://www.zdf.de', 'http://www.filmweb.pl', 'http://www.eenadu.net', 'http://www.lichess.org', 'http://www.yoox.com', 'http://www.mobiletracking.ru', 'http://www.itmedia.co.jp', 'http://www.bestbinary.site', 'http://www.getbootstrap.com', 'http://www.wemakeprice.com', 'http://www.maybank2u.com.my', 'http://www.xuite.net', 'http://www.illinois.edu', 'http://www.pearsoncmg.com', 'http://www.berkeley.edu', 'http://www.bodybuilding.com', 'http://www.dlsite.com', 'http://www.gimmemore.com', 'http://www.torrentfreak.com', 'http://www.bmi.ir', 'http://www.searchdimension.com', 'http://www.lostfilmhd.ru', 'http://www.fetlife.com', 'http://www.riotgames.com', 'http://www.rabb.it', 'http://www.glosbe.com', 'http://www.vz.ru', 'http://www.techtudo.com.br', 'http://www.24h.com.vn', 'http://www.habr.com', 'http://www.oxforddictionaries.com', 'http://www.pansubscribe.com', 'http://www.lamoda.ru', 'http://www.sfgate.com', 'http://www.freelancer.com', 'http://www.brazzersnetwork.com', 'http://www.sketchup.com', 'http://www.blackfriday.com', 'http://www.ontvtime.ru', 'http://www.t-mobile.com', 'http://www.mp3party.net', 'http://www.filmfanatic.com', 'http://www.pearsoned.com', 'http://www.kompasiana.com', 'http://www.mangareader.net', 'http://www.nate.com', 'http://www.pcmag.com', 'http://www.matchtv.ru', 'http://www.wired.com', 'http://www.repelis.live', 'http://www.punchng.com', 'http://www.hdrezka.ag', 'http://www.soccer-live.stream', 'http://www.orf.at', 'http://www.cisco.com', 'http://www.finn.no', 'http://www.baixaki.com.br', 'http://www.canada.ca', 'http://www.rockstargames.com', 'http://www.xhamsterpremium.com', 'http://www.rp5.ru', 'http://www.carcc.net', 'http://www.rosegal.com', 'http://www.sheypoor.com', 'http://www.rightmove.co.uk', 'http://www.custhelp.com', 'http://www.imagebam.com', 'http://www.payu.in', 'http://www.variety.com', 'http://www.f95zone.com', 'http://www.clicknupload.org', 'http://www.downloadastro.com', 'http://www.zoomit.ir', 'http://www.r18.com', 'http://www.ria.com', 'http://www.lofter.com', 'http://www.digitalocean.com', 'http://www.suara.com', 'http://www.pdfdrive.com', 'http://www.leo.org', 'http://www.immobilienscout24.de', 'http://www.panet.co.il', 'http://www.yllanorin.com', 'http://www.rakuten-card.co.jp', 'http://www.ahlmasrnews.com', 'http://www.brainly.co.id', 'http://www.ceneo.pl', 'http://www.newsweek.com', 'http://www.sci-hub.tw', 'http://www.com-fast-macbook.live', 'http://www.dribbble.com', 'http://www.icy-veins.com', 'http://www.twoo.com', 'http://www.eset.com', 'http://www.mehrnews.com', 'http://www.smugmug.com', 'http://www.sportingvideo.com', 'http://www.8maple.ru', 'http://www.mysearch.com', 'http://www.elespanol.com', 'http://www.thedailybeast.com', 'http://www.rotoworld.com', 'http://www.caixa.gov.br', 'http://www.semanticscholar.org', 'http://www.turnitin.com', 'http://www.tvtropes.org', 'http://www.torrentdownloads.me', 'http://www.ranker.com', 'http://www.livemaster.ru', 'http://www.victoriassecret.com', 'http://www.arabi21.com', 'http://www.tokyomotion.net', 'http://www.umich.edu', 'http://www.tmofans.com', 'http://www.z1.fm', 'http://www.verizon.com', 'http://www.formula1.com', 'http://www.gutefrage.net', 'http://www.makeleio.gr', 'http://www.blacktiestreams.xyz', 'http://www.pelispedia.tv', 'http://www.lavanguardia.com', 'http://www.elgenero.com', 'http://www.zee5.com', 'http://www.britishcouncil.org', 'http://www.theweathernetwork.com', 'http://www.ana.co.jp', 'http://www.faptitans.com', 'http://www.syosetu.com', 'http://www.flightradar24.com', 'http://www.usbank.com', 'http://www.phimmoi.net', 'http://www.kinogo.by', 'http://www.smi2.ru', 'http://www.schoology.com', 'http://www.epfindia.gov.in', 'http://www.nikkansports.com', 'http://www.royalbank.com', 'http://www.getadblock.com', 'http://www.chatwork.com', 'http://www.9anime.to', 'http://www.nmisr.com', 'http://www.provincial.com', 'http://www.comandotorrents.com', 'http://www.wikibooks.org', 'http://www.moneycontrol.com', 'http://www.mynavi.jp', 'http://www.sakura.ne.jp', 'http://www.unrealengine.com', 'http://www.exlibrisgroup.com', 'http://www.techcrunch.com', 'http://www.rojadirecta.me', 'http://www.readthedocs.io', 'http://www.rg.ru', 'http://www.4cdn.org', 'http://www.dpreview.com', 'http://www.symbolab.com', 'http://www.so-net.ne.jp', 'http://www.nbc.com', 'http://www.24smi.info', 'http://www.zukxd6fkxqn.com', 'http://www.wisc.edu', 'http://www.kenh14.vn', 'http://www.amd.com', 'http://www.proxybay.bz', 'http://www.daraz.com.bd', 'http://www.bluehost.com', 'http://www.poste.it', 'http://www.samsclub.com', 'http://www.overstock.com', 'http://www.polygon.com', 'http://www.searchencrypt.com', 'http://www.wondemninge.info', 'http://www.elintransigente.com', 'http://www.2gok8g15p2.com', 'http://www.meteofrance.com', 'http://www.turbo.az', 'http://www.digitalspy.com', 'http://www.01net.com', 'http://www.clickfunnels.com', 'http://www.qiita.com', 'http://www.tap.az', 'http://www.sakshi.com', 'http://www.thepiratebay-proxylist.se', 'http://www.clckads.org', 'http://www.cbslocal.com', 'http://www.privatbank.ua', 'http://www.ufreegames.com', 'http://www.gazeta.pl', 'http://www.baskino.me', 'http://www.adidas.com', 'http://www.lostfilm-hd720.ru', 'http://www.donanimhaber.com', 'http://www.cornell.edu', 'http://www.epn.bz', 'http://www.definicion.de', 'http://www.islcollective.com', 'http://www.crhoy.com', 'http://www.informationvine.com', 'http://www.onliner.by', 'http://www.tenki.jp', 'http://www.ebscohost.com', 'http://www.hibids10.com', 'http://www.autotrader.com', 'http://www.livescores.com', 'http://www.herokuapp.com', 'http://www.billboard.com', 'http://www.20minutos.es', 'http://www.enstage-sas.com', 'http://www.tutsplus.com', 'http://www.lg.com', 'http://www.watchonlinemovies.com.pk', 'http://www.folha.uol.com.br', 'http://www.ilfattoquotidiano.it', 'http://www.marketwatch.com', 'http://www.divxtotal3.net', 'http://www.womanadvice.ru', 'http://www.logitech.com', 'http://www.match.com', 'http://www.elcomercio.com', 'http://www.mcafee.com', 'http://www.narcity.com', 'http://www.vtv.vn', 'http://www.fortnitetracker.com', 'http://www.mediamarkt.de', 'http://www.todayhumor.co.kr', 'http://www.nptel.ac.in', 'http://www.gofundme.com', 'http://www.tsetmc.com', 'http://www.businessweekly.com.tw', 'http://www.lidl.de', 'http://www.zi.media', 'http://www.voyeurhit.com', 'http://www.atlassian.net', 'http://www.starzplay.com', 'http://www.coub.com', 'http://www.rtve.es', 'http://www.tabloidbintang.com', 'http://www.unity.com', 'http://www.vg.no', 'http://www.ucla.edu', 'http://www.walgreens.com', 'http://www.oneindia.com', 'http://www.watchcartoononline.io', 'http://www.adveric.net', 'http://www.bolasport.com', 'http://www.columbia.edu', 'http://www.myfitnesspal.com', 'http://www.pathofexile.com', 'http://www.rvcj.com', 'http://www.fmkorea.com', 'http://www.marketgid.com', 'http://www.deezer.com', 'http://www.unity3d.com', 'http://www.pnc.com', 'http://www.mts.ru', 'http://www.postbank.de', 'http://www.bab.la', 'http://www.thebalancecareers.com', 'http://www.oload.stream', 'http://www.pixlr.com', 'http://www.lib1.org', 'http://www.fbdown.net', 'http://www.azet.sk', 'http://www.eurogamer.net', 'http://www.news18.com', 'http://www.cuevana2.com', 'http://www.wykskkpplgfi.com', 'http://www.wikiwiki.jp', 'http://www.cameraprive.com', 'http://www.eldorado.ru', 'http://www.3dlat.com', 'http://www.hotukdeals.com', 'http://www.metro.co.uk', 'http://www.kongregate.com', 'http://www.2gis.ru', 'http://www.yggtorrent.to', 'http://www.intellitest.me', 'http://www.check24.de', 'http://www.plex.tv', 'http://www.wowkeren.com', 'http://www.virtualworldsland.com', 'http://www.tradedoubler.com', 'http://www.tagesschau.de', 'http://www.thenewsminute.com', 'http://www.kalerkantho.com', 'http://www.gameforge.com', 'http://www.biggo.com.tw', 'http://www.literotica.com', 'http://www.jcrew.com', 'http://www.bahn.de', 'http://www.wizards.com', 'http://www.maam.ru', 'http://www.mydealz.de', 'http://www.dnvod.tv', 'http://www.altervista.org', 'http://www.pochta.ru', 'http://www.cosmopolitan.com', 'http://www.meijutt.com', 'http://www.freebitco.in', 'http://www.astrologyanswers.com', 'http://www.ilbe.com', 'http://www.ubuntu.com', 'http://www.lolesports.com', 'http://www.consultant.ru', 'http://www.iobit.com', 'http://www.gradeup.co', 'http://www.virgilio.it', 'http://www.iptorrents.com', 'http://www.dailysnark.com', 'http://www.thefappeningblog.com', 'http://www.spanishdict.com', 'http://www.niniban.com', 'http://www.aif.ru', 'http://www.turkiye.gov.tr', 'http://www.forever21.com', 'http://www.hostgator.com', 'http://www.emag.ro', 'http://www.n-torrents.org', 'http://www.willhaben.at', 'http://www.secondlife.com', 'http://www.worldfree4u.club', 'http://www.yourdictionary.com', 'http://www.bitbucket.org', 'http://www.hstqqjxqwnrfhy.com', 'http://www.skidrowreloaded.com', 'http://www.bedbathandbeyond.com', 'http://www.deref-gmx.net', 'http://www.mundodeportivo.com', 'http://www.gooool.org', 'http://www.vente-privee.com', 'http://www.att.net', 'http://www.techtarget.com', 'http://www.reclameaqui.com.br', 'http://www.who.int', 'http://www.tiu.ru', 'http://www.utexas.edu', 'http://www.livestrong.com', 'http://www.eluniverso.com', 'http://www.howstuffworks.com', 'http://www.vulture.com', 'http://www.joins.com', 'http://www.mashable.com', 'http://www.nyu.edu', 'http://www.buzzfeednews.com', 'http://www.abema.tv', 'http://www.joom.com', 'http://www.index-education.net', 'http://www.qvc.com', 'http://www.videolan.org', 'http://www.mathxl.com', 'http://www.mirtesen.ru', 'http://www.boardgamegeek.com', 'http://www.obozrevatel.com', 'http://www.manganelo.com', 'http://www.tmearn.com', 'http://www.dailypost.ng', 'http://www.eltima.com', 'http://www.almaany.com', 'http://www.bestblackfriday.com', 'http://www.tiki.vn', 'http://www.gtarcade.com', 'http://www.pbs.org', 'http://www.click-now-extra-special.online', 'http://www.edmodo.com', 'http://www.nationalgeographic.com', 'http://www.noaa.gov', 'http://www.androidcentral.com', 'http://www.standardmedia.co.ke', 'http://www.commbank.com.au', 'http://www.theqoo.net', 'http://www.editor.wix.com', 'http://www.argos.co.uk', 'http://www.testbook.com', 'http://www.tabnak.ir', 'http://www.utoronto.ca', 'http://www.newyorker.com', 'http://www.amazon.com.br', 'http://www.lordfilm.tv', 'http://www.nnm-club.me', 'http://www.kora-star.tv', 'http://www.subhd.com', 'http://www.alwatanvoice.com', 'http://www.kshow123.net', 'http://www.hindustantimes.com', 'http://www.allbest.ru', 'http://www.draftkings.com', 'http://www.usenet.nl', 'http://www.alhilalalyoum.com', 'http://www.swagbucks.com', 'http://www.jawabkom.com', 'http://www.20minutes.fr', 'http://www.qoo10.sg', 'http://www.qz.com', 'http://www.ynet.co.il', 'http://www.ubisoft.com', 'http://www.instiz.net', 'http://www.mover.uz', 'http://www.llbean.com', 'http://www.cdc.gov', 'http://www.moddb.com', 'http://www.twogap.com', 'http://www.riafan.ru', 'http://www.cyberleninka.ru', 'http://www.brainly.com.br', 'http://www.rajasthan.gov.in', 'http://www.credit-agricole.fr', 'http://www.kakprosto.ru', 'http://www.wetter.com', 'http://www.binomo.com', 'http://www.heise.de', 'http://www.definition.org', 'http://www.ifixit.com', 'http://www.kotaku.com', 'http://www.olx.ro', 'http://www.rei.com', 'http://www.visualstudio.com', 'http://www.irs.gov', 'http://www.tamilgun.rocks', 'http://www.kora-online.tv', 'http://www.washington.edu', 'http://www.predictionds.com', 'http://www.kopilkaurokov.ru', 'http://www.talk4anyone.com', 'http://www.medianewpage.com', 'http://www.thepopple.com', 'http://www.wikisource.org', 'http://www.afrigatenews.net', 'http://www.amazon.com.au', 'http://www.doordash.com', 'http://www.papajohns.com', 'http://www.larepublica.pe', 'http://www.lanacion.com.ar', 'http://www.viralcpm.com', 'http://www.onedio.com', 'http://www.emojipedia.org', 'http://www.gemius.pl', 'http://www.similarweb.com', 'http://www.khmerload.com', 'http://www.duden.de', 'http://www.ntaneet.nic.in', 'http://www.roll20.net', 'http://www.garmin.com', 'http://www.cvs.com', 'http://www.ssc.nic.in', 'http://www.tamin.ir', 'http://www.gamesradar.com', 'http://www.yesmovies.to', 'http://www.labirint.ru', 'http://www.sankakucomplex.com', 'http://www.filmix.co', 'http://www.refinery29.com', 'http://www.chasethetrend.com', 'http://www.rusvesna.su', 'http://www.bola.com', 'http://www.pinterest.ru', 'http://www.el-nacional.com', 'http://www.basketball-reference.com', 'http://www.elconfidencial.com', 'http://www.atwiki.jp', 'http://www.fastpic.ru', 'http://www.sportzbonanza.com', 'http://www.poshmark.com', 'http://www.dreamstime.com', 'http://www.un.org', 'http://www.58b.tv', 'http://www.blog.ir', 'http://www.crunchbase.com', 'http://www.olx.com.eg', 'http://www.joinhoney.com', 'http://www.alphacoders.com', 'http://www.laprensagrafica.com', 'http://www.padlet.com', 'http://www.betradar.com', 'http://www.fextralife.com', 'http://www.thekitchn.com', 'http://www.chosun.com', 'http://www.mellowads.com', 'http://www.autoscout24.de', 'http://www.metric-conversions.org', 'http://www.time.ir', 'http://www.kbb.com', 'http://www.jalan.net', 'http://www.justwatch.com', 'http://www.slideplayer.com', 'http://www.razer.com', 'http://www.thevintagenews.com', 'http://www.kapook.com', 'http://www.sparknotes.com', 'http://www.leparisien.fr', 'http://www.zeit.de', 'http://www.indiatoday.in', 'http://www.haberturk.com', 'http://www.maplestage.com', 'http://www.qwant.com', 'http://www.shopee.ph', 'http://www.freedownloadmanager.org', 'http://www.solarmoviez.ru', 'http://www.kotobank.jp', 'http://www.arga-mag.com', 'http://www.masterani.me', 'http://www.nifty.com', 'http://www.tstartel.com', 'http://www.blocket.se', 'http://www.intoday.in', 'http://www.yr.no', 'http://www.disq.us', 'http://www.linguee.fr', 'http://www.arenavision.link', 'http://www.softpedia.com', 'http://www.houzz.com', 'http://www.bol.com', 'http://www.deviantart.net', 'http://www.apartments.com', 'http://www.emalls.ir', 'http://www.javmost.com', 'http://www.tripadvisor.co.uk', 'http://www.malwarebytes.com', 'http://www.to10.gr', 'http://www.umn.edu', 'http://www.csfd.cz', 'http://www.aftonbladet.se', 'http://www.soft98.ir', 'http://www.impots.gouv.fr', 'http://www.origin.com', 'http://www.userbenchmark.com', 'http://www.comicbook.com', 'http://www.php.net', 'http://www.anime-sugoi.com', 'http://www.porngames.adult', 'http://www.depor.com', 'http://www.filelist.ro', 'http://www.ozbargain.com.au', 'http://www.myfood.ltd', 'http://www.ets.org', 'http://www.adf.ly', 'http://www.utarget.ru', 'http://www.hdfilmcehennemi2.org', 'http://www.sportmail.ru', 'http://www.zerohedge.com', 'http://www.xn--i1abbnckbmcl9fb.xn--p1ai', 'http://www.mosreg.ru', 'http://www.farfeshplus.com', 'http://www.i.ua', 'http://www.icims.com', 'http://www.thesimsresource.com', 'http://www.hotpepper.jp', 'http://www.sbrf.ru', 'http://www.eoredi.com', 'http://www.ency-education.com', 'http://www.stripe.com', 'http://www.stockx.com', 'http://www.ennaharonline.com', 'http://www.clubic.com', 'http://www.javdoe.com', 'http://www.opensooq.com', 'http://www.uchi.ru', 'http://www.funnyjunk.com', 'http://www.mackolik.com', 'http://www.arstechnica.com', 'http://www.chefkoch.de', 'http://www.informer.com', 'http://www.bama.ir', 'http://www.realestate.com.au', 'http://www.sbnation.com', 'http://www.npmjs.com', 'http://www.zozo.jp', 'http://www.ppomppu.co.kr', 'http://www.pentapostagma.gr', 'http://www.reimageplus.com', 'http://www.whitepages.com', 'http://www.barnesandnoble.com', 'http://www.bestbuy.ca', 'http://www.myshared.ru', 'http://www.computerbild.de', 'http://www.givemp3.com', 'http://www.bulbagarden.net', 'http://www.easeus.com', 'http://www.medscape.com', 'http://www.pch.com', 'http://www.bethesda.net', 'http://www.disney.go.com', 'http://www.gettyimages.com', 'http://www.appssupply.net', 'http://www.sanook.com', 'http://www.24smi.org', 'http://www.jetbrains.com', 'http://www.futhead.com', 'http://www.miniclip.com', 'http://www.colourpop.com', 'http://www.myegy.cc', 'http://www.greatandhra.com', 'http://www.24video.sexy', 'http://www.gogoanimes.tv', 'http://www.si.com', 'http://www.listindiario.com', 'http://www.initiativeq.com', 'http://www.sportzwiki.com', 'http://www.pornxs.com', 'http://www.trafficfactory.biz', 'http://www.privat24.ua', 'http://www.filgoal.com', 'http://www.khaberni.com', 'http://www.gleam.io', 'http://www.cars.com', 'http://www.unam.mx', 'http://www.screenrant.com', 'http://www.deref-web-02.de', 'http://www.arduino.cc', 'http://www.cmu.edu', 'http://www.jumia.com.eg', 'http://www.qiwi.com', 'http://www.grailed.com', 'http://www.ilmeteo.it', 'http://www.matchat.online', 'http://www.woman.ru', 'http://www.dr.dk', 'http://www.blockchain.com', 'http://www.payoneer.com', 'http://www.fantasti.cc', 'http://www.hds.to', 'http://www.elbotola.com', 'http://www.adorocinema.com', 'http://www.dummies.com', 'http://www.gigazine.net', 'http://www.masterads.info', 'http://www.pelisplus.co', 'http://www.megabonus.com', 'http://www.safefinder.com', 'http://www.o2.pl', 'http://www.ghanaweb.com', 'http://www.hackstore.net', 'http://www.planetsuzy.org', 'http://www.infusionsoft.com', 'http://www.haberler.com', 'http://www.vmware.com', 'http://www.videohive.net', 'http://www.lego.com', 'http://www.slate.com', 'http://www.usc.edu', 'http://www.sport24.gr', 'http://www.eonline.com', 'http://www.ixbt.com', 'http://www.axisbank.com', 'http://www.workfromhomejobsonline.co', 'http://www.aranzulla.it', 'http://www.fanduel.com', 'http://www.junbi-tracker.com', 'http://www.o2tvseries.com', 'http://www.sciencemag.org', 'http://www.beinmatch.com', 'http://www.khabarfarsi.com', 'http://www.easports.com', 'http://www.soccer365.ru', 'http://www.etherscan.io', 'http://www.index.hu', 'http://www.upenn.edu', 'http://www.ulta.com', 'http://www.woot.com', 'http://www.tvn24.pl', 'http://www.familysearch.org', 'http://www.ukzn.ac.za', 'http://www.caf.fr', 'http://www.nitroflare.com', 'http://www.informereng.com', 'http://www.jollychic.com', 'http://www.easybib.com', 'http://www.nesn.com', 'http://www.kino-filmi.net', 'http://www.gamib.net', 'http://www.roosterteeth.com', 'http://www.fool.com', 'http://www.esquire.com', 'http://www.tvguide.com', 'http://www.nikkeibp.co.jp', 'http://www.hpjav.tv', 'http://www.sarayanews.com', 'http://www.meaww.com', 'http://www.windowscentral.com', 'http://www.ballotpedia.org', 'http://www.udacity.com', 'http://www.ouest-france.fr', 'http://www.mydiba.one', 'http://www.sport.es', 'http://www.cas.sk', 'http://www.thrillist.com', 'http://www.sport-express.ru', 'http://www.hollywoodreporter.com', 'http://www.ratemyprofessors.com', 'http://www.foxsports.com', 'http://www.gucci.com', 'http://www.ilacrehberi.com', 'http://www.moneymake.site', 'http://www.pelisplus.to', 'http://www.ttmeiju.vip', 'http://www.special-offers.online', 'http://www.tesla.com', 'http://www.kaggle.com', 'http://www.coinbase.com', 'http://www.stubhub.com', 'http://www.larousse.fr', 'http://www.alc.co.jp', 'http://www.www.gob.mx', 'http://www.zulily.com', 'http://www.myvidster.com', 'http://www.livelib.ru', 'http://www.ezgif.com', 'http://www.babycenter.com', 'http://www.timeout.com', 'http://www.tripadvisor.ru', 'http://www.sweetwater.com', 'http://www.goibibo.com', 'http://www.scribol.com', 'http://www.pandora.tv', 'http://www.multitran.ru', 'http://www.tparser.org', 'http://www.hipwee.com', 'http://www.statista.com', 'http://www.brainyquote.com', 'http://www.biglobe.ne.jp', 'http://www.lynda.com', 'http://www.juntadeandalucia.es', 'http://www.trafyield.com', 'http://www.dotabuff.com', 'http://www.ansa.it', 'http://www.sabah.com.tr', 'http://www.priceline.com', 'http://www.nalog.ru', 'http://www.swiggy.com', 'http://www.koreastardaily.com', 'http://www.nessma.tv', 'http://www.720p-izle.com', 'http://www.mercadolibre.com', 'http://www.mamba.ru', 'http://www.indiegogo.com', 'http://www.sueddeutsche.de', 'http://www.funsafetab.com', 'http://www.muzofond.fm', 'http://www.farpost.ru', 'http://www.anime1.me', 'http://www.bankersadda.com', 'http://www.yandex.ua', 'http://www.users.wix.com', 'http://www.alexa.com', 'http://www.arte.tv', 'http://www.ssiapawz.com', 'http://www.uloz.to', 'http://www.haber7.com', 'http://www.iz.ru', 'http://www.crazygames.com', 'http://www.entrepreneur.com', 'http://www.itch.io', 'http://www.gaitheed.com', 'http://www.schwab.com', 'http://www.jsmcrjmp.com', 'http://www.consumerreports.org', 'http://www.cafebazaar.ir', 'http://www.happyon.jp', 'http://www.obrazovaka.ru', 'http://www.inc.com', 'http://www.ae.com', 'http://www.500px.com', 'http://www.techbang.com', 'http://www.oknews.site', 'http://www.xvideos.blog.br', 'http://www.streamcloud.eu', 'http://www.extratorrent.ag', 'http://www.globalnews.ca', 'http://www.1337x.st', 'http://www.tass.ru', 'http://www.laleggepertutti.it', 'http://www.oui.sncf', 'http://www.apache.org', 'http://www.foozine.com', 'http://www.dikaiologitika.gr', 'http://www.mydramalist.com', 'http://www.sarkariexam.com', 'http://www.twirpx.com', 'http://www.endclothing.com', 'http://www.moviefone.com', 'http://www.ceehimur.uk', 'http://www.vultr.com', 'http://www.80s.tw', 'http://www.sprint.com', 'http://www.letterboxd.com', 'http://www.kooora2day.com', 'http://www.sh.st', 'http://www.minuto30.com', 'http://www.aglasem.com', 'http://www.turkiyegazetesi.com.tr', 'http://www.smh.com.au', 'http://www.sears.com', 'http://www.sportsdirect.com', 'http://www.vodafone.de', 'http://www.commentcamarche.com', 'http://www.elcorteingles.es', 'http://www.shikimori.org', 'http://www.filmibeat.com', 'http://www.22pixx.xyz', 'http://www.crackwatch.com', 'http://www.russia.tv', 'http://www.passplay.com', 'http://www.vanguard.com', 'http://www.filimo.com', 'http://www.icook.tw', 'http://www.sinefight-skinesia.com', 'http://www.macrumors.com', 'http://www.mediawhirl.net', 'http://www.somoynews.tv', 'http://www.bd-pratidin.com', 'http://www.olx.co.id', 'http://www.hilton.com', 'http://www.putlockers.co', 'http://www.staples.com', 'http://www.monova.to', 'http://www.chewy.com', 'http://www.oyi9f1kbaj.com', 'http://www.exblog.jp', 'http://www.pornohype.net', 'http://www.readmanga.me', 'http://www.medicalnewstoday.com', 'http://www.getlnk6.com', 'http://www.ubc.ca', 'http://www.kensaq.com', 'http://www.wetteronline.de', 'http://www.teachable.com', 'http://www.laposte.net', 'http://www.inoreader.com', 'http://www.worldoftanks.eu', 'http://www.scotiabank.com', 'http://www.healthfirstnews.com', 'http://www.doodle.com', 'http://www.mediaworld.it', 'http://www.bhaskar.com', 'http://www.javarchive.com', 'http://www.legacy.com', 'http://www.aweber.com', 'http://www.nouvelobs.com', 'http://www.rt.ru', 'http://www.movavi.com', 'http://www.seekingalpha.com', 'http://www.macmillandictionary.com', 'http://www.esporte.uol.com.br', 'http://www.rockpapershotgun.com', 'http://www.colorado.edu', 'http://www.pagenews.gr', 'http://www.popmyads.com', 'http://www.wykop.pl', 'http://www.russian7.ru', 'http://www.dmv.org', 'http://www.shiftdelete.net', 'http://www.bangbros.com', 'http://www.kemdikbud.go.id', 'http://www.rappler.com', 'http://www.cima4up.tv', 'http://www.timesnownews.com', 'http://www.kinoprofi.org', 'http://www.gazzetta.gr', 'http://www.movieweb.com', 'http://www.jogos360.com.br', 'http://www.trademe.co.nz', 'http://www.olx.kz', 'http://www.planetromeo.com', 'http://www.parivahan.gov.in', 'http://www.cosmo.ru', 'http://www.yektanet.com', 'http://www.smarter.com', 'http://www.adright.co', 'http://www.laposte.fr', 'http://www.brainly.lat', 'http://www.beeline.ru', 'http://www.bfmtv.com', 'http://www.adsclk.com', 'http://www.kinogo2.cc', 'http://www.soccerway.com', 'http://www.bu.edu', 'http://www.popdaily.com.tw', 'http://www.mangadex.org', 'http://www.apnews.com', 'http://www.videouroki.net', 'http://www.foursquare.com', 'http://www.daypush.com', 'http://www.ucdavis.edu', 'http://www.hugedomains.com', 'http://www.topshape.me', 'http://www.asu.edu', 'http://www.price.com.hk', 'http://www.ny.gov', 'http://www.cifraclub.com.br', 'http://www.walmart.ca', 'http://www.streamcomplet.me', 'http://www.javatpoint.com', 'http://www.history.com', 'http://www.24tv.ua', 'http://www.epa.gov', 'http://www.fantasypros.com', 'http://www.barclaycardus.com', 'http://www.lavasoft.com', 'http://www.olx.pt', 'http://www.bbcgoodfood.com', 'http://www.squareup.com', 'http://www.tolivelugu.com', 'http://www.leetcode.com', 'http://www.warcraftlogs.com', 'http://www.truecaller.com', 'http://www.nymag.com', 'http://www.codecademy.com', 'http://www.2ch.hk', 'http://www.po-kaki-to.com', 'http://www.olx.in', 'http://www.037hd.com', 'http://www.hamgardi.com', 'http://www.stoiximan.gr', 'http://www.indosport.com', 'http://www.eurosport.fr', 'http://www.myscore.ru', 'http://www.my-hit.org', 'http://www.cafesiyaset.com.tr', 'http://www.emuparadise.me', 'http://www.newgrounds.com', 'http://www.dhl.de', 'http://www.msu.edu', 'http://www.creativemarket.com', 'http://www.plarium.com', 'http://www.notification-browser.com', 'http://www.dickssportinggoods.com', 'http://www.anidub.com', 'http://www.caffeinamagazine.it', 'http://www.piaotian.com', 'http://www.seesaa.net', 'http://www.zamunda.net', 'http://www.medlineplus.gov', 'http://www.coostuni.com', 'http://www.gigya.com', 'http://www.momovod.com', 'http://www.biblehub.com', 'http://www.twitcasting.tv', 'http://www.tripadvisor.in', 'http://www.glaz.tv', 'http://www.6pm.com', 'http://www.geforce.com', 'http://www.shmoop.com', 'http://www.jumia.ma', 'http://www.pwieu.com', 'http://www.dexchangeinc.com', 'http://www.ucoz.net', 'http://www.allcalidad.com', 'http://www.atlassian.com', 'http://www.elsalvador.com', 'http://www.megaup.net', 'http://www.mejortorrent1.com', 'http://www.zerodha.com', 'http://www.amctheatres.com', 'http://www.rollingstone.com', 'http://www.telechargerunevideo.com', 'http://www.thomsonreuters.com', 'http://www.couchtuner.cloud', 'http://www.medu.ir', 'http://www.ew.com', 'http://www.sporcle.com', 'http://www.gulte.com', 'http://www.topstreams.info', 'http://www.poe.trade', 'http://www.mojang.com', 'http://www.bongacams2.com', 'http://www.naijaloaded.com.ng', 'http://www.autotrader.co.uk', 'http://www.rivals.com', 'http://www.infoseek.co.jp', 'http://www.nation-news.ru', 'http://www.typeform.com', 'http://www.programiz.com', 'http://www.storage.googleapis.com', 'http://www.cutestat.com', 'http://www.quikr.com', 'http://www.irib.ir', 'http://www.fmovies.io', 'http://www.eodev.com', 'http://www.rutgers.edu', 'http://www.shareasale.com', 'http://www.e-katalog.ru', 'http://www.runescape.com', 'http://www.opentable.com', 'http://www.pornosveta.com', 'http://www.shareba.com', 'http://www.zalando.de', 'http://www.firstrowsportes.net', 'http://www.tasteofhome.com', 'http://www.kanobu.ru', 'http://www.linternaute.com', 'http://www.resetera.com', 'http://www.vivino.com', 'http://www.90tv.ir', 'http://www.torrentday.com', 'http://www.gamewith.jp', 'http://www.pipspread.com', 'http://www.mashreghnews.ir', 'http://www.tirto.id', 'http://www.louisvuitton.com', 'http://www.einthusan.tv', 'http://www.cian.ru', 'http://www.linguee.com', 'http://www.hibapress.com', 'http://www.tgd.kr', 'http://www.easyjet.com', 'http://www.usg.edu', 'http://www.idealista.com', 'http://www.vokrug.tv', 'http://www.stern.de', 'http://www.lyricstranslate.com', 'http://www.passportindia.gov.in', 'http://www.sbt.com.br', 'http://www.adobelogin.com', 'http://www.deepl.com', 'http://www.vetogate.com', 'http://www.hdpopcorns.co', 'http://www.tsn.ua', 'http://www.vipergirls.to', 'http://www.battlefield.com', 'http://www.droom.in', 'http://www.edmunds.com', 'http://www.planetminecraft.com', 'http://www.emirates.com', 'http://www.lightinthebox.com', 'http://www.agar.io', 'http://www.tilestwra.com', 'http://www.rpfonlinereg.org', 'http://www.copts-united.com', 'http://www.pcworld.com', 'http://www.greasyfork.org', 'http://www.dict.cc', 'http://www.freshdesk.com', 'http://www.lifedaily.com', 'http://www.wmtransfer.com', 'http://www.scielo.br', 'http://www.cic.gc.ca', 'http://www.hsbc.com.hk', 'http://www.lodynet.com', 'http://www.tetew.info', 'http://www.euroki.org', 'http://www.ox.ac.uk', 'http://www.thequestion.ru', 'http://www.javqd.com', 'http://www.temp-mail.org', 'http://www.downloadhub.link', 'http://www.smallseotools.com', 'http://www.glassdoor.co.in', 'http://www.funimation.com', 'http://www.1111.com.tw', 'http://www.cuny.edu', 'http://www.index.hr', 'http://www.collegeboard.org', 'http://www.livesoccertv.com', 'http://www.afrtrk.com', 'http://www.marmiton.org', 'http://www.tudotv.tv', 'http://www.mylivecricket.info', 'http://www.caixabank.es', 'http://www.ru-clip.net', 'http://www.greget.space', 'http://www.firefox.com', 'http://www.ziprecruiter.com', 'http://www.tcafe2a.com', 'http://www.yle.fi', 'http://www.lenovo.com.cn', 'http://www.azgyimccolyyo.com', 'http://www.raider.io', 'http://www.manofile.com', 'http://www.filezog.com', 'http://www.onlinekhabar.com', 'http://www.jal.co.jp', 'http://www.katfile.com', 'http://www.pohs2oom.com', 'http://www.gumtree.pl', 'http://www.worldbank.org', 'http://www.tureng.com', 'http://www.hsbc.co.uk', 'http://www.virginmedia.com', 'http://www.irna.ir', 'http://www.calameo.com', 'http://www.gogoanime.in', 'http://www.iwara.tv', 'http://www.teespring.com', 'http://www.downloadtwittervideo.com', 'http://www.ksu.edu.sa', 'http://www.hi.ru', 'http://www.fragrantica.com', 'http://www.xbooks.to', 'http://www.rateyourmusic.com', 'http://www.fitbit.com', 'http://www.tunein.com', 'http://www.crichd.com', 'http://www.mthai.com', 'http://www.speed-open2.com', 'http://www.tpi.it', 'http://www.terra.com.br', 'http://www.imore.com', 'http://www.gitlab.com', 'http://www.yandex.net', 'http://www.jagran.com', 'http://www.unc.edu', 'http://www.nerdwallet.com', 'http://www.anthropologie.com', 'http://www.jabong.com', 'http://www.france.tv', 'http://www.jpmorganchase.com', 'http://www.enotes.com', 'http://www.photobucket.com', 'http://www.putrr16.com', 'http://www.brainly.in', 'http://www.constantcontact.com', 'http://www.x-minus.me', 'http://www.webmoney.ru', 'http://www.olx.com.pk', 'http://www.add0n.com', 'http://www.kommersant.ru', 'http://www.static1.squarespace.com', 'http://www.infusionsoft.app', 'http://www.infox.sg', 'http://www.gq.com', 'http://www.daftsex.com', 'http://www.eslamoda.com', 'http://www.theringer.com', 'http://www.xataka.com', 'http://www.bitmedianetwork.com', 'http://www.allkpop.com', 'http://www.kulichki.net', 'http://www.crictracker.com', 'http://www.shein.com.mx', 'http://www.brightspace.com', 'http://www.itslearning.com', 'http://www.mismarcadores.com', 'http://www.bartarinha.ir', 'http://www.freecodecamp.org', 'http://www.proquest.com', 'http://www.samplicio.us', 'http://www.libertyvf.org', 'http://www.excite.co.jp', 'http://www.dresslily.com', 'http://www.xero.com', 'http://www.kinos.to', 'http://www.linuxmint.com', 'http://www.tuasaude.com', 'http://www.libgen.pw', 'http://www.11bet.com', 'http://www.qatarliving.com', 'http://www.neilpatel.com', 'http://www.trip.com', 'http://www.www-guard.com', 'http://www.publinews.gt', 'http://www.convertio.co', 'http://www.bizrate.com', 'http://www.sbicard.com', 'http://www.guancha.cn', 'http://www.blackdoctor.org', 'http://www.pagesjaunes.fr', 'http://www.shahid4u.me', 'http://www.imgsrc.ru', 'http://www.yummyanime.com', 'http://www.nxctrk.com', 'http://www.soy502.com', 'http://www.abv.bg', 'http://www.diablo3.com', 'http://www.kompoz.me', 'http://www.globalsources.com', 'http://www.gumtree.co.za', 'http://www.genndi.com', 'http://www.1und1.de', 'http://www.casasbahia.com.br', 'http://www.tes.com', 'http://www.dek-d.com', 'http://www.alza.cz', 'http://www.gemius.com', 'http://www.jalopnik.com', 'http://www.korrespondent.net', 'http://www.duniadigest.com', 'http://www.brasilescola.uol.com.br', 'http://www.jpnn.com', 'http://www.citethisforme.com', 'http://www.bovada.lv', 'http://www.cheapoair.com', 'http://www.shop-apotheke.at', 'http://www.aviasales.ru', 'http://www.todamateria.com.br', 'http://www.okaz.com.sa', 'http://www.ncore.cc', 'http://www.cox.net', 'http://www.passeidireto.com', 'http://www.anten.ir', 'http://www.ft.com', 'http://www.pearsonmylabandmastering.com', 'http://www.djelfa.info', 'http://www.delish.com', 'http://www.elkhabar.com', 'http://www.jut.su', 'http://www.joq.al', 'http://www.starbucks.com', 'http://www.lifeder.com', 'http://www.monster.com', 'http://www.jang.com.pk', 'http://www.pizzahut.com', 'http://www.www.nhs.uk', 'http://www.webnovel.com', 'http://www.mydiv.net', 'http://www.offerup.com', 'http://www.theculturetrip.com', 'http://www.express.pk', 'http://www.tankionline.com', 'http://www.thespruce.com', 'http://www.salamdl.info', 'http://www.lowyat.net', 'http://www.vogue.com.tw', 'http://www.chetor.com', 'http://www.bd24live.com', 'http://www.forocoches.com', 'http://www.notifychheck.com', 'http://www.dailythanthi.com', 'http://www.shrinkearn.com', 'http://www.rarbgmirror.org', 'http://www.normour.com', 'http://www.goody25.com', 'http://www.qdownloader.net', 'http://www.creditmutuel.fr', 'http://www.seriouseats.com', 'http://www.tinhte.vn', 'http://www.bouyguestelecom.fr', 'http://www.eltiempo.es', 'http://www.aljazeera.net', 'http://www.abadis.ir', 'http://www.mysql.com', 'http://www.reshak.ru', 'http://www.dream.co.id', 'http://www.diaforetiko.gr', 'http://www.3djuegos.com', 'http://www.india.com', 'http://www.btdx8.com', 'http://www.bazos.sk', 'http://www.imgbox.com', 'http://www.thespruceeats.com', 'http://www.nsw.gov.au', 'http://www.fixya.com', 'http://www.ffmovies.ru', 'http://www.mango.com', 'http://www.gftnk.com', 'http://www.dcfever.com', 'http://www.saturn.de', 'http://www.dostor.org', 'http://www.magazineluiza.com.br', 'http://www.sfchronicle.com', 'http://www.uci.edu', 'http://www.evite.com', 'http://www.tf1.fr', 'http://www.chomikuj.pl', 'http://www.hanime.tv', 'http://www.b-ok.cc', 'http://www.standardnews.com', 'http://www.mastercard.com.au', 'http://www.qatarairways.com', 'http://www.wday.ru', 'http://www.kantipurdaily.com', 'http://www.wolfram.com', 'http://www.get-mp3.me', 'http://www.gaoqing.la', 'http://www.cplusplus.com', 'http://www.hln.be', 'http://www.assia.tv', 'http://www.rbcroyalbank.com', 'http://www.fruithosts.net', 'http://www.infocenter.support', 'http://www.hackerrank.com', 'http://www.southcn.com', 'http://www.newsnow.co.uk', 'http://www.drivemusic.me', 'http://www.shahrekhabar.com', 'http://www.livescience.com', 'http://www.michaels.com', 'http://www.2hanwriten.com', 'http://www.donya-e-eqtesad.com', 'http://www.thenorthface.com', 'http://www.omegle.com', 'http://www.dantri.com.vn', 'http://www.kadu.ru', 'http://www.umd.edu', 'http://www.fossbytes.com', 'http://www.ufc.com', 'http://www.online-audio-converter.com', 'http://www.marcaapuestas.es', 'http://www.eldiario.es', 'http://www.mobileadsserver.com', 'http://www.mercadolivre.com', 'http://www.encuentra24.com', 'http://www.skyscanner.com', 'http://www.successfactors.com', 'http://www.spaste.com', 'http://www.garena.com', 'http://www.tamilwin.com', 'http://www.autozone.com', 'http://www.eater.com', 'http://www.citibank.co.in', 'http://www.mwave.me', 'http://www.rankedboost.com', 'http://www.indiarailinfo.com', 'http://www.girrrly.com', 'http://www.androidauthority.com', 'http://www.musixmatch.com', 'http://www.answers.com', 'http://www.pdf2doc.com', 'http://www.thechive.com', 'http://www.shareae.com', 'http://www.grand-casino50.com', 'http://www.panorama.com.al', 'http://www.komplett.no', 'http://www.eltiempo.com', 'http://www.light-dark.net', 'http://www.ss.com', 'http://www.netgear.com', 'http://www.mentalfloss.com', 'http://www.lankasri.com', 'http://www.dl.free.fr', 'http://www.shareasale-analytics.com', 'http://www.s3.com.tw', 'http://www.famousbirthdays.com', 'http://www.graucoay.net', 'http://www.scoop.it', 'http://www.codeproject.com', 'http://www.areavip.com.br', 'http://www.tooxclusive.com', 'http://www.desire2learn.com', 'http://www.megaresheba.ru', 'http://www.thisisinsider.com', 'http://www.netkeiba.com', 'http://www.megafon.ru', 'http://www.torob.com', 'http://www.seek.com.au', 'http://www.lolcounter.com', 'http://www.piliapp.com', 'http://www.eurosport.ru', 'http://www.suamusica.com.br', 'http://www.elnuevodiario.com.ni', 'http://www.tecmundo.com.br', 'http://www.musescore.com', 'http://www.kamupersoneli.net', 'http://www.realclearpolitics.com', 'http://www.jobrapido.com', 'http://www.loverslab.com', 'http://www.protothema.gr', 'http://www.mexashare.com', 'http://www.etajakhabar.com', 'http://www.hotmart.com', 'http://www.netacad.com', 'http://www.vemale.com', 'http://www.chicagotribune.com', 'http://www.telugu360.com', 'http://www.abc7.com', 'http://www.budsgunshop.com', 'http://www.nex1music.ir', 'http://www.boxofficemojo.com', 'http://www.kurir.rs', 'http://www.dji.com', 'http://www.linternaute.fr', 'http://www.darty.com', 'http://www.vocabulary.com', 'http://www.cibercuba.com', 'http://www.2ch-c.net', 'http://www.infowars.com', 'http://www.chordify.net', 'http://www.alwakeelnews.com', 'http://www.swzz.xyz', 'http://www.hentai2read.com', 'http://www.vivo.sx', 'http://www.abc.go.com', 'http://www.bookdepository.com', 'http://www.mixer.com', 'http://www.rarbgmirror.xyz', 'http://www.blic.rs', 'http://www.adda247.com', 'http://www.zalukaj.com', 'http://www.ssg.com', 'http://www.tomshardware.co.uk', 'http://www.best2018games.com', 'http://www.russianfood.com', 'http://www.santander.co.uk', 'http://www.flashresultats.fr', 'http://www.weheartit.com', 'http://www.acer.com', 'http://www.hirufm.lk', 'http://www.mixcloud.com', 'http://www.cyberforum.ru', 'http://www.musica.com', 'http://www.shopbop.com', 'http://www.librus.pl', 'http://www.eluniversal.com.mx', 'http://www.gadgetsnow.com', 'http://www.wooordhunt.ru', 'http://www.zoznam.sk', 'http://www.vitalsource.com', 'http://www.freenet.de', 'http://www.warhistoryonline.com', 'http://www.researchnow.com', 'http://www.dailypakistan.com.pk', 'http://www.o12zs3u2n.com', 'http://www.suumo.jp', 'http://www.transferwise.com', 'http://www.yale.edu', 'http://www.grubhub.com', 'http://www.lloydsbank.co.uk', 'http://www.laptopmag.com', 'http://www.tubeoffline.com', 'http://www.magnetdl.com', 'http://www.ssense.com', 'http://www.digiato.com', 'http://www.openclassrooms.com', 'http://www.rzd.ru', 'http://www.uznayvse.ru', 'http://www.amarujala.com', 'http://www.joinindianarmy.nic.in', 'http://www.stream2watch.org', 'http://www.wolframalpha.com', 'http://www.ebay.ie', 'http://www.cleartrip.com', 'http://www.wikidot.com', 'http://www.radiko.jp', 'http://www.dicio.com.br', 'http://www.leagueofgraphs.com', 'http://www.manualslib.com', 'http://www.expressen.se', 'http://www.btdb.to', 'http://www.8ch.net', 'http://www.gamestorrents.tv', 'http://www.1plus1tv.ru', 'http://www.marksandspencer.com', 'http://www.geekbuying.com', 'http://www.lastpass.com', 'http://www.altadefinizione.fm', 'http://www.geniuskitchen.com', 'http://www.islamweb.net', 'http://www.24livenewspaper.com', 'http://www.ccn.com', 'http://www.bpi.ir', 'http://www.clickbank.com', 'http://www.buyee.jp', 'http://www.kaspersky.com', 'http://www.vsco.co', 'http://www.planetwin365.it', 'http://www.teletica.com', 'http://www.sensacine.com', 'http://www.vidoza.net', 'http://www.alternativeto.net', 'http://www.softbank.jp', 'http://www.cwtv.com', 'http://www.navyfederal.org', 'http://www.wunderlist.com', 'http://www.parsfootball.com', 'http://www.alarab.com', 'http://www.loboclick.com', 'http://www.whoscored.com', 'http://www.cybersource.com', 'http://www.boredpanda.com', 'http://www.jutarnji.hr', 'http://www.onlinemoviewatch.org', 'http://www.zmz003.com', 'http://www.stoloto.ru', 'http://www.jisho.org', 'http://www.ournewstoday.com', 'http://www.fao.org', 'http://www.sling.com', 'http://www.puu.sh', 'http://www.cuntwars.com', 'http://www.tsite.jp', 'http://www.afkarnews.com', 'http://www.aznude.com', 'http://www.gamepressure.com', 'http://www.ycombinator.com', 'http://www.serebii.net', 'http://www.all-free-download.com', 'http://www.dugtor.org', 'http://www.psicologiaymente.com', 'http://www.strawpoll.me', 'http://www.snopes.com', 'http://www.taroot-rangi.com', 'http://www.vshkole.com', 'http://www.desmos.com', 'http://www.kooralive.info', 'http://www.serialssolutions.com', 'http://www.clk.icu', 'http://www.klikbca.com', 'http://www.sportmaster.ru', 'http://www.jokerlivestream.com', 'http://www.cloudinary.com', 'http://www.tp-link.com', 'http://www.mp4upload.com', 'http://www.topporno.tv', 'http://www.bc.vc', 'http://www.stihi.ru', 'http://www.monotaro.com', 'http://www.dizibox.pw', 'http://www.template.net', 'http://www.oricon.co.jp', 'http://www.vistaprint.com', 'http://www.tukif.com', 'http://www.zone-telechargement.lol', 'http://www.computerhope.com', 'http://www.hkjc.com', 'http://www.kohsantepheapdaily.com.kh', 'http://www.mindbodyonline.com', 'http://www.ccavenue.com', 'http://www.ksl.com', 'http://www.slader.com', 'http://www.9tsu.com', 'http://www.css-tricks.com', 'http://www.bdupload.info', 'http://www.internet-start.net', 'http://www.powerschool.com', 'http://www.offerzone.click', 'http://www.gostream.site', 'http://www.at.ua', 'http://www.craigslist.ca', 'http://www.gqbuzz.com', 'http://www.coupons.com', 'http://www.jusbrasil.com.br', 'http://www.spaggiari.eu', 'http://www.complex.com', 'http://www.acestream.net', 'http://www.matchesfashion.com', 'http://www.cracked.com', 'http://www.coolmathgames.com', 'http://www.guildwars2.com', 'http://www.cuon.io', 'http://www.ssisurveys.com', 'http://www.statusklic.info', 'http://www.gsu.edu', 'http://www.ren.tv', 'http://www.flightaware.com', 'http://www.todoist.com', 'http://www.aporrea.org', 'http://www.uchicago.edu', 'http://www.shopstyle.com', 'http://www.webassign.net', 'http://www.rojadirectatv.tv', 'http://www.curiouscat.me', 'http://www.mint.com', 'http://www.ondramanice.co', 'http://www.picofile.com', 'http://www.gta5-mods.com', 'http://www.ardmediathek.de', 'http://www.sabay.com.kh', 'http://www.colorlib.com', 'http://www.wallapop.com', 'http://www.pushnice.com', 'http://www.citibankonline.com', 'http://www.hentai-foundry.com', 'http://www.infoescola.com', 'http://www.oagudsey.uk', 'http://www.faceit.com', 'http://www.deutsche-bank.de', 'http://www.share-online.biz', 'http://www.prensalibre.com', 'http://www.leroymerlin.ru', 'http://www.pichunter.com', 'http://www.gocomics.com', 'http://www.shatel.ir', 'http://www.tvp.pl', 'http://www.iu.edu', 'http://www.kino-hd720.net', 'http://www.proboards.com', 'http://www.rstyle.me', 'http://www.photios-raj.com', 'http://www.indiapost.gov.in', 'http://www.lcl.fr', 'http://www.graphicriver.net', 'http://www.segodnya.ua', 'http://www.arcgis.com', 'http://www.pluralsight.com', 'http://www.mapquest.com', 'http://www.kshowonline.com', 'http://www.puma.com', 'http://www.express.com.pk', 'http://www.moneysavingexpert.com', 'http://www.mycanal.fr', 'http://www.fbcdn2.com', 'http://www.filmesonlineseries.net', 'http://www.catal0g.info', 'http://www.torrentigruha.ru', 'http://www.uw.edu', 'http://www.mtggoldfish.com', 'http://www.unacademy.com', 'http://www.scholastic.com', 'http://www.ntv.ru', 'http://www.minecraftforum.net', 'http://www.hentaiheroes.com', 'http://www.zimbio.com', 'http://www.eghtesadnews.com', 'http://www.jav.guru', 'http://www.oberlo.com', 'http://www.cityheaven.net', 'http://www.unique-tutorials.info', 'http://www.diariolibre.com', 'http://www.parimatch.com', 'http://www.bluewin.ch', 'http://www.latam.com', 'http://www.ref-dir.com', 'http://www.dysfz.vip', 'http://www.ufl.edu', 'http://www.websiteoutlook.com', 'http://www.vijesti.ba', 'http://www.tutu.ru', 'http://www.lyst.com', 'http://www.fmovie.cc', 'http://www.today.com', 'http://www.commonapp.org', 'http://www.mebook.cc', 'http://www.popsugar.com', 'http://www.usgamer.net', 'http://www.fosshub.com', 'http://www.ibb.co', 'http://www.hypebeast.com', 'http://www.postimg.cc', 'http://www.heureka.cz', 'http://www.ismedia.jp', 'http://www.gobizkorea.com', 'http://www.dailystar.co.uk', 'http://www.noticias.uol.com.br', 'http://www.medianewpagesearch.com', 'http://www.lastampa.it', 'http://www.bankbazaar.com', 'http://www.post.ir', 'http://www.addtoany.com', 'http://www.ipko.pl', 'http://www.yatra.com', 'http://www.extramovies.trade', 'http://www.zvuk.com', 'http://www.doramy.club', 'http://www.wwe.com', 'http://www.vidal.fr', 'http://www.gnavi.co.jp', 'http://www.lacuerda.net', 'http://www.presslogic.com', 'http://www.fakaza.com', 'http://www.caisse-epargne.fr', 'http://www.superanimes.site', 'http://www.concursolutions.com', 'http://www.scroll.in', 'http://www.telekom.com', 'http://www.deskgram.net', 'http://www.netshoes.com.br', 'http://www.soft112.com', 'http://www.globaltestmarket.com', 'http://www.fresherslive.com', 'http://www.moveadrenaline.com', 'http://www.zimbra.free.fr', 'http://www.momomall.com.tw', 'http://www.23andme.com', 'http://www.movistarplus.es', 'http://www.dagbladet.no', 'http://www.mhometheater.com', 'http://www.stuff.co.nz', 'http://www.issetdigital.com', 'http://www.adbtc.top', 'http://www.solvusoft.com', 'http://www.docker.com', 'http://www.donationalerts.ru', 'http://www.gatech.edu', 'http://www.sport24.ru', 'http://www.acm.org', 'http://www.dktaqipmquo.com', 'http://www.kaishist.top', 'http://www.popjav.tv', 'http://www.square-enix.com', 'http://www.skyscanner.ru', 'http://www.bioskopkeren.fun', 'http://www.northwestern.edu', 'http://www.new-rutor.org', 'http://www.cam.ac.uk', 'http://www.giga.de', 'http://www.alarabiya.net', 'http://www.marinetraffic.com', 'http://www.unicredit.it', 'http://www.world4ufree.vip', 'http://www.nsaem.net', 'http://www.yaklass.ru', 'http://www.filejoker.net', 'http://www.hgtv.com', 'http://www.seemorgh.com', 'http://www.olmsoneenh.info', 'http://www.sarzamindownload.com', 'http://www.rakyatku.com', 'http://www.hesport.com', 'http://www.yes24.com', 'http://www.almowaten.net', 'http://www.mathsisfun.com', 'http://www.ethz.ch', 'http://www.futbol24.com', 'http://www.elle.com', 'http://www.4kdownload.com', 'http://www.shazam.com', 'http://www.topky.sk', 'http://www.tabici.com', 'http://www.cineulagam.com', 'http://www.animelek.net', 'http://www.mdpi.com', 'http://www.wikiquote.org', 'http://www.bobaedream.co.kr', 'http://www.streamplay.to', 'http://www.supersport.com', 'http://www.futura-sciences.com', 'http://www.anwap.film', 'http://www.nextdirect.com', 'http://www.meduza.io', 'http://www.vrbo.com', 'http://www.imagevenue.com', 'http://www.vercomicsporno.com', 'http://www.ceconline.com', 'http://www.top1health.com', 'http://www.retiremely.com', 'http://www.curbed.com', 'http://www.heraldweekly.com', 'http://www.callofduty.com', 'http://www.megadede.com', 'http://www.princeton.edu', 'http://www.carousell.com', 'http://www.mosaiquefm.net', 'http://www.oclc.org', 'http://www.higherperspectives.com', 'http://www.go2too.com', 'http://www.here.com', 'http://www.jezebel.com', 'http://www.rarbgmirror.com', 'http://www.kanald.com.tr', 'http://www.uploadboy.com', 'http://www.songkick.com', 'http://www.flixtor.to', 'http://www.ihg.com', 'http://www.lonelyplanet.com', 'http://www.solvetube.site', 'http://www.correios.com.br', 'http://www.usgs.gov', 'http://www.mcgill.ca', 'http://www.imgflip.com', 'http://www.aovivonatv.com', 'http://www.kinohronik.net', 'http://www.gdeposylka.ru', 'http://www.soccer24.com', 'http://www.toyokeizai.net', 'http://www.readcomiconline.to', 'http://www.download-space.com', 'http://www.kickassanime.io', 'http://www.tesco.com', 'http://www.gosunoob.com', 'http://www.wallpaperscraft.com', 'http://www.akhbarona.com', 'http://www.jetstar.com', 'http://www.jhu.edu', 'http://www.le360.ma', 'http://www.destyy.com', 'http://www.sherdog.com', 'http://www.wpbeginner.com', 'http://www.ecured.cu', 'http://www.bigporn.com', 'http://www.verywellhealth.com', 'http://www.delfi.lv', 'http://www.liveadexchanger.com', 'http://www.virginia.edu', 'http://www.newsru.com', 'http://www.azcentral.com', 'http://www.legendei.com', 'http://www.info.com', 'http://www.pitchfork.com', 'http://www.didestan.com', 'http://www.duke.edu', 'http://www.everydayhealth.com', 'http://www.studbooks.net', 'http://www.naughtyamerica.com', 'http://www.xxxdan.com', 'http://www.getcourse.ru', 'http://www.usaweekly.com', 'http://www.startimes.com', 'http://www.proxyof.com', 'http://www.collegedunia.com', 'http://www.moneyforward.com', 'http://www.tvanimemuryoudouga.com', 'http://www.cybozu.com', 'http://www.yemek.com', 'http://www.emeraldinsight.com', 'http://www.republika.co.id', 'http://www.neimanmarcus.com', 'http://www.trustnav.com', 'http://www.1movies.is', 'http://www.99kubo.tv', 'http://www.torrents.me', 'http://www.elbilad.net', 'http://www.wetter.de', 'http://www.tamilyogi.cc', 'http://www.rtvonline.com', 'http://www.bellesa.co', 'http://www.sifyitest.com', 'http://www.sbs.com.au', 'http://www.archdaily.com', 'http://www.ultipro.com', 'http://www.vecteezy.com', 'http://www.loteriasyapuestas.es', 'http://www.coolrom.com', 'http://www.songsterr.com', 'http://www.nextyourcontent.com', 'http://www.televisionparatodos.tv', 'http://www.moviesrulzfree.com', 'http://www.ravelry.com', 'http://www.detmir.ru', 'http://www.inbox.lv', 'http://www.cox.com', 'http://www.bigcartel.com', 'http://www.baomoi.com', 'http://www.jbhifi.com.au', 'http://www.ing-diba.de', 'http://www.earthlink.net', 'http://www.crictime.com', 'http://www.sima-land.ru', 'http://www.sport-fm.gr', 'http://www.akhbarelyom.com', 'http://www.pc-torrent.games', 'http://www.orbitz.com', 'http://www.theknot.com', 'http://www.dailywire.com', 'http://www.kroger.com', 'http://www.brightside.me', 'http://www.mercadolibre.cl', 'http://www.alamy.com', 'http://www.doctissimo.fr', 'http://www.frtyi.com', 'http://www.overdrive.com', 'http://www.techwalla.com', 'http://www.escapadarural.com', 'http://www.canlitv.plus', 'http://www.life.ru', 'http://www.suruga-ya.jp', 'http://www.aeroflot.ru', 'http://www.medianetto.com', 'http://www.firstsrowsports.tv', 'http://www.crateandbarrel.com', 'http://www.tsdm.me', 'http://www.dinamalar.com', 'http://www.actcorp.in', 'http://www.frys.com', 'http://www.hitcpm.com', 'http://www.mytedata.net', 'http://www.varzesh11.com', 'http://www.itau.com.br', 'http://www.instacart.com', 'http://www.zvooq.online', 'http://www.repretel.com', 'http://www.fnb.co.za', 'http://www.palermotoday.it', 'http://www.picmonkey.com', 'http://www.daraz.lk', 'http://www.westelm.com', 'http://www.sportradar.com', 'http://www.societegenerale.fr', 'http://www.onlyfans.com', 'http://www.rapidtables.com', 'http://www.u.gg', 'http://www.upsc.gov.in', 'http://www.indiewire.com', 'http://www.vectorstock.com', 'http://www.serverfault.com', 'http://www.clicknshare.net', 'http://www.takealot.com', 'http://www.rsc.org', 'http://www.sitepoint.com', 'http://www.menards.com', 'http://www.os.tc', 'http://www.11street.my', 'http://www.record.pt', 'http://www.commerzbank.de', 'http://www.xpau.se', 'http://www.tiscali.it', 'http://www.raspberrypi.org', 'http://www.ojooo.com', 'http://www.al3omk.com', 'http://www.joyreactor.cc', 'http://www.guitarcenter.com', 'http://www.christian-dogma.com', 'http://www.s.to', 'http://www.wyborcza.pl', 'http://www.javfinder.ru', 'http://www.vrv.co', 'http://www.estrenosdoramas.net', 'http://www.streams.tv', 'http://www.faradars.org', 'http://www.piratebay.com', 'http://www.oreilly.com', 'http://www.masrawy.com', 'http://www.crichd.to', 'http://www.nps.gov', 'http://www.xn--j1ahfl.xn--p1ai', 'http://www.uwatchfree.se', 'http://www.beatport.com', 'http://www.dongtw.com', 'http://www.intellicast.com', 'http://www.rarbgaccess.org', 'http://www.zimuku.cn', 'http://www.scipy.org', 'http://www.fanfox.net', 'http://www.weathernews.jp', 'http://www.oyunkolu.com', 'http://www.education.com', 'http://www.marieclaire.com.tw', 'http://www.linguee.de', 'http://www.animesonlinebr.com.br', 'http://www.besoccer.com', 'http://www.zergnet.com', 'http://www.sammobile.com', 'http://www.sinonimos.com.br', 'http://www.oriflame.com', 'http://www.nydailynews.com', 'http://www.kotak.com', 'http://www.gmx.at', 'http://www.dailykos.com', 'http://www.hangseng.com', 'http://www.tripadvisor.it', 'http://www.narvar.com', 'http://www.realcenter-mobileapps1.com', 'http://www.yandex.uz', 'http://www.mangapanda.com', 'http://www.slither.io', 'http://www.tfile.me', 'http://www.imagecurl.com', 'http://www.receitasdedoces.info', 'http://www.amalgama-lab.com', 'http://www.geico.com', 'http://www.ucf.edu', 'http://www.entekhab.ir', 'http://www.game-game.com.ua', 'http://www.yonhapnews.co.kr', 'http://www.cint.com', 'http://www.rokna.net', 'http://www.gridoto.com', 'http://www.kimcartoon.to', 'http://www.britishairways.com', 'http://www.hd-1080.com', 'http://www.upsconline.nic.in', 'http://www.deadline.com', 'http://www.thewindowsclub.com', 'http://www.slrclub.com', 'http://www.csod.com', 'http://www.wondershare.net', 'http://www.xpornplease.com', 'http://www.lnk.to', 'http://www.huffingtonpost.fr', 'http://www.theconversation.com', 'http://www.retre.org', 'http://www.ut.ac.ir', 'http://www.lufthansa.com', 'http://www.easyziptab.com', 'http://www.estadao.com.br', 'http://www.vg247.com', 'http://www.biblestudytools.com', 'http://www.submarino.com.br', 'http://www.nhaccuatui.com', 'http://www.city-data.com', 'http://www.walla.co.il', 'http://www.portalroms.com', 'http://www.sapo.ao', 'http://www.rojadirecta.eu', 'http://www.puhutv.com', 'http://www.drugs.com', 'http://www.bazdeh.org', 'http://www.tvefamosos.uol.com.br', 'http://www.angel.co', 'http://www.thetrainline.com', 'http://www.pcgamesn.com', 'http://www.megogo.net', 'http://www.misionesonline.net', 'http://www.desafiomundial.com', 'http://www.vt.edu', 'http://www.leafly.com', 'http://www.extreme-d0wn.com', 'http://www.immobiliare.it', 'http://www.animefreak.tv', 'http://www.stream-cr7.net', 'http://www.bignox.com', 'http://www.resultados.com', 'http://www.litmir.me', 'http://www.torlock.com', 'http://www.traveloka.com', 'http://www.ulmart.ru', 'http://www.laredoute.fr', 'http://www.robertsspaceindustries.com', 'http://www.cebupacificair.com', 'http://www.blu-ray.com', 'http://www.delfi.lt', 'http://www.rlsbb.ru', 'http://www.significados.com.br', 'http://www.kolkata24x7.com', 'http://www.akhbarak.net', 'http://www.hi5.com', 'http://www.zoopla.co.uk', 'http://www.valshara.com', 'http://www.wikireading.ru', 'http://www.greenmangaming.com', 'http://www.avira.com', 'http://www.yesstyle.com', 'http://www.origo.hu', 'http://www.steemit.com', 'http://www.zezoomglobal.com', 'http://www.secure-surf.net', 'http://www.carview.co.jp', 'http://www.service-now.com', 'http://www.volafile.org', 'http://www.tsn.ca', 'http://www.staseraintv.com', 'http://www.epnclick.ru', 'http://www.ignou.ac.in', 'http://www.bbva.es', 'http://www.novelupdates.com', 'http://www.3dnews.ru', 'http://www.tupaki.com', 'http://www.ubereats.com', 'http://www.yemeksepeti.com', 'http://www.rezultati.com', 'http://www.anistar.me', 'http://www.soompi.com', 'http://www.icai.org', 'http://www.foxsportsla.com', 'http://www.kinoly.net', 'http://www.softcatalog.info', 'http://www.amazon.com.tr', 'http://www.veopornogratis.xxx', 'http://www.jetblue.com', 'http://www.notizie.it', 'http://www.ef.com', 'http://www.filmcomplet.la', 'http://www.smadav.net', 'http://www.dt2xr6g2i5.com', 'http://www.go2cloud.org', 'http://www.dunyanews.tv', 'http://www.filehorse.com', 'http://www.meta.ua', 'http://www.gds.it', 'http://www.juegosonce.es', 'http://www.smotrisport.tv', 'http://www.worldoftanks.com', 'http://www.vanityfair.com', 'http://www.mydirtyhobby.com', 'http://www.myutiitsl.com', 'http://www.yahoo.com.tw', 'http://www.pcgames-download.com', 'http://www.parsnaz.com', 'http://www.cimbclicks.com.my', 'http://www.microcenter.com', 'http://www.bmj.com', 'http://www.mrporter.com', 'http://www.indiabix.com', 'http://www.camwhores.tv', 'http://www.tawk.to', 'http://www.watch2gether.com', 'http://www.mericasads.com', 'http://www.ameli.fr', 'http://www.tracku02.com', 'http://www.myzcloud.me', 'http://www.findagrave.com', 'http://www.car.gr', 'http://www.idernzastoeok.com', 'http://www.accenture.com', 'http://www.sina.com.tw', 'http://www.pbih.org', 'http://www.rome2rio.com', 'http://www.indeed.co.in', 'http://www.clipwatching.com', 'http://www.srf.ch', 'http://www.syl.ru', 'http://www.fril.jp', 'http://www.express.com', 'http://www.businessnewslive2018.online', 'http://www.jquery.com', 'http://www.bigmir.net', 'http://www.justporno.sex', 'http://www.fontawesome.com', 'http://www.kijiji.it', 'http://www.nrzhlsvqxbgpbn.com', 'http://www.mexgroup.com', 'http://www.commercialvalue.org', 'http://www.nu.nl', 'http://www.zoom.com.br', 'http://www.pro-football-reference.com', 'http://www.gadgetnews.net', 'http://www.fortune.com', 'http://www.systemrequirementslab.com', 'http://www.lexilogos.com', 'http://www.asianwiki.com', 'http://www.zazzle.com', 'http://www.vagaro.com', 'http://www.shein.in', 'http://www.hearthpwn.com', 'http://www.tvonline.plus', 'http://www.nesine.com', 'http://www.airbnb.fr', 'http://www.alriyadh.news', 'http://www.watchserieshd.io', 'http://www.tamu.edu', 'http://www.ummy.net', 'http://www.hnonline.sk', 'http://www.pensador.com', 'http://www.dota2.com', 'http://www.eprice.com.tw', 'http://www.ctvnews.ca', 'http://www.armorgames.com', 'http://www.laravel.com', 'http://www.costco.ca', 'http://www.theadvocate.com', 'http://www.wikifeet.com', 'http://www.aps.org', 'http://www.jackpotchances.com', 'http://www.basnews.com', 'http://www.deloitte.com', 'http://www.cnscg.com', 'http://www.cardinalcommerce.com', 'http://www.studwood.ru', 'http://www.hdlava.me', 'http://www.popcornvod.com', 'http://www.itv.com', 'http://www.gogafilm.me', 'http://www.netbarg.com', 'http://www.nvsp.in', 'http://www.yumpu.com', 'http://www.playsport.cc', 'http://www.bmo.com', 'http://www.playvk.com', 'http://www.regmarkets.ru', 'http://www.oploverz.in', 'http://www.hindilinks4u.to', 'http://www.travelocity.com', 'http://www.rentalcars.com', 'http://www.mydigit.cn', 'http://www.msnbc.com', 'http://www.cabelas.com', 'http://www.addic7ed.com', 'http://www.svyaznoy.ru', 'http://www.ceskatelevize.cz', 'http://www.turbosquid.com', 'http://www.shop.com.mm', 'http://www.2nn.jp', 'http://www.ssa.gov', 'http://www.anadolu.edu.tr', 'http://www.w3.org', 'http://www.r7.com', 'http://www.1337x.one', 'http://www.instant-gaming.com', 'http://www.huffpost.com', 'http://www.soha.vn', 'http://www.thepiratebay.online', 'http://www.99acres.com', 'http://www.hbogo.com', 'http://www.football.ua', 'http://www.reifefrauen-date.com', 'http://www.rootgames.org', 'http://www.probiller.com', 'http://www.9minecraft.net', 'http://www.adyen.com', 'http://www.jsfiddle.net', 'http://www.blender.org', 'http://www.chia-anime.tv', 'http://www.aoredi.com', 'http://www.ellitoral.com', 'http://www.drp.su', 'http://www.regnum.ru', 'http://www.megapesni.me', 'http://www.potterybarn.com', 'http://www.piazza.com', 'http://www.ursportslife.com', 'http://www.careerpower.in', 'http://www.pirateproxy.live', 'http://www.mobile.ir', 'http://www.ford.com', 'http://www.ofeetles.pro', 'http://www.delgarm.com', 'http://www.20min.ch', 'http://www.guarrasdelporno.xxx', 'http://www.thewirecutter.com', 'http://www.service-public.fr', 'http://www.jd.ru', 'http://www.krone.at', 'http://www.vseinstrumenti.ru', 'http://www.katmoviehd.tv', 'http://www.privalia.com', 'http://www.bilutv.net', 'http://www.panasonic.com', 'http://www.onlinemultfilmy.ru', 'http://www.inflibnet.ac.in', 'http://www.canadiantire.ca', 'http://www.notebookcheck.net', 'http://www.freecharge.in', 'http://www.protonmail.com', 'http://www.cafe24.com', 'http://www.zdnet.com', 'http://www.dhl.com', 'http://www.festyy.com', 'http://www.tribune.com.pk', 'http://www.newsd.co', 'http://www.sidereel.com', 'http://www.24sata.hr', 'http://www.fidonav.com', 'http://www.humoruniv.com', 'http://www.epicurious.com', 'http://www.hbr.org', 'http://www.nitori-net.jp', 'http://www.thebalance.com', 'http://www.scientificamerican.com', 'http://www.sarkariresults.org.in', 'http://www.inosmi.ru', 'http://www.teipgupp.com', 'http://www.mazika2day.com', 'http://www.pickfilm.ru', 'http://www.fuxoasim.link', 'http://www.mobipo.org', 'http://www.pccomponentes.com', 'http://www.mightytext.net', 'http://www.money.pl', 'http://www.ped-kopilka.ru', 'http://www.livehindustan.com', 'http://www.thairath.co.th', 'http://www.drivereasy.com', 'http://www.homes.co.jp', 'http://www.codechef.com', 'http://www.subf2m.co', 'http://www.banglanews24.com', 'http://www.000webhostapp.com', 'http://www.0123movies.com', 'http://www.goindigo.in', 'http://www.pozdravok.ru', 'http://www.ctui.in', 'http://www.anz.com', 'http://www.todaypk.stream', 'http://www.k2nblog.com', 'http://www.maharashtra.gov.in', 'http://www.disney.com', 'http://www.linustechtips.com', 'http://www.putlocker.vip', 'http://www.aljazeera.com', 'http://www.alaskaair.com', 'http://www.gametop.com', 'http://www.hdfilmcehennemi1.com', 'http://www.image-line.com', 'http://www.mudah.my', 'http://www.piraeuspress.gr', 'http://www.nchsoftware.com', 'http://www.mp3cc.biz', 'http://www.kaiserpermanente.org', 'http://www.malavida.com', 'http://www.shaamtv.com', 'http://www.cont.ws', 'http://www.wankoz.com', 'http://www.cool3c.com', 'http://www.windowsreport.com', 'http://www.compucalitv.com', 'http://www.esam.ir', 'http://www.thetimes.co.uk', 'http://www.vagina.nl', 'http://www.kabum.com.br', 'http://www.uptodate.com', 'http://www.kulturologia.ru', 'http://www.getfvid.com', 'http://www.officedepot.com', 'http://www.memrise.com', 'http://www.veporns.com', 'http://www.srfiles.com', 'http://www.code.org', 'http://www.moi.gov.qa', 'http://www.docusign.net', 'http://www.brndrm.com', 'http://www.gktoday.in', 'http://www.kinosimka.net', 'http://www.warning.support', 'http://www.typepad.com', 'http://www.bittrex.com', 'http://www.currys.co.uk', 'http://www.kahoot.it', 'http://www.bancadigitalbod.com', 'http://www.anime-planet.com', 'http://www.mxforexctr.online', 'http://www.e-reading.club', 'http://www.moudamepo.com', 'http://www.bestdateshere22.com', 'http://www.unsw.edu.au', 'http://www.mirrored.to', 'http://www.news24.com', 'http://www.next.co.uk', 'http://www.yify-torrent.org', 'http://www.avidreaders.ru', 'http://www.torrentdownload.ch', 'http://www.yellowpages.com', 'http://www.soccervista.com', 'http://www.nieuwsblad.be', 'http://www.nine.com.au', 'http://www.iastate.edu', 'http://www.aquipelis.tv', 'http://www.eclipse.org', 'http://www.nametests.com', 'http://www.superhaber.tv', 'http://www.bershka.com', 'http://www.ebalka.cc', 'http://www.torrentsave.xyz', 'http://www.skillshare.com', 'http://www.bible.com', 'http://www.moviejoa.net', 'http://www.philenews.com', 'http://www.homepros.zone', 'http://www.amtrak.com', 'http://www.1plus1.ua', 'http://www.immowelt.de', 'http://www.vatanbilgisayar.com', 'http://www.net-a-porter.com', 'http://www.pearson.com', 'http://www.axios.com', 'http://www.skidrow-games.com', 'http://www.fardanews.com', 'http://www.dl-protect1.com', 'http://www.homemaking.com', 'http://www.liuli.eu', 'http://www.redirect2719.ws', 'http://www.sport.ua', 'http://www.talk.tw', 'http://www.fararu.com', 'http://www.firstpost.com', 'http://www.olx.co.ao', 'http://www.nexon.net', 'http://www.sofort.com', 'http://www.sdna.gr', 'http://www.derstandard.at', 'http://www.ekstrabladet.dk', 'http://www.jplovetv.com', 'http://www.mci.ir', 'http://www.osu.edu', 'http://www.ally.com', 'http://www.khamsat.com', 'http://www.uchitelya.com', 'http://www.torrentdosfilmeshd.net', 'http://www.hastidl.me', 'http://www.washingtontimes.com', 'http://www.streamplay.me', 'http://www.dream11.com', 'http://www.myheritage.com', 'http://www.gelocal.it', 'http://www.mintmanga.com', 'http://www.sogi.com.tw', 'http://www.dealmoon.com', 'http://www.hatenablog.jp', 'http://www.bunnings.com.au', 'http://www.vietnamnet.vn', 'http://www.cibc.com', 'http://www.boulanger.com', 'http://www.animesync.tv', 'http://www.tvplusnewtabsearch.com', 'http://www.apornoonlain.tv', 'http://www.wittyfeed.tv', 'http://www.bazos.cz', 'http://www.vodafone.in', 'http://www.dkb.de', 'http://www.balkanweb.com', 'http://www.dvdvideosoft.com', 'http://www.msftconnecttest.com', 'http://www.pottermore.com', 'http://www.umass.edu', 'http://www.fastcompany.com', 'http://www.johnlewis.com', 'http://www.2ememain.be', 'http://www.gezginler.net', 'http://www.joybuy.com', 'http://www.collider.com', 'http://www.quirkybyte.com', 'http://www.selfridges.com', 'http://www.producthunt.com', 'http://www.patagonia.com', 'http://www.freebunker.com', 'http://www.iheart.com', 'http://www.standard.co.uk', 'http://www.abplive.in', 'http://www.readfree.me', 'http://www.canalblog.com', 'http://www.optimum.net', 'http://www.calculator.net', 'http://www.bankrate.com', 'http://www.zapmeta.ws', 'http://www.nosub.tv', 'http://www.todaytvseries2.com', 'http://www.lilo.org', 'http://www.akakce.com', 'http://www.avnori.pro', 'http://www.socratic.org', 'http://www.footyroom.com', 'http://www.getlnk9.com', 'http://www.gestyy.com', 'http://www.jisutiyu.com', 'http://www.letgo.com', 'http://www.lumpics.ru', 'http://www.whatmobile.com.pk', 'http://www.eklablog.com', 'http://www.templatemonster.com', 'http://www.offersuperhub.com', 'http://www.about.com', 'http://www.tbs.co.jp', 'http://www.gogy.com', 'http://www.iitk.ac.in', 'http://www.time.is', 'http://www.recruit.co.jp', 'http://www.pitt.edu', 'http://www.alquds.co.uk', 'http://www.gva.es', 'http://www.jn-news.com', 'http://www.arzdigital.com', 'http://www.1001fonts.com', 'http://www.utah.edu', 'http://www.nodejs.org', 'http://www.bartleby.com', 'http://www.datacamp.com', 'http://www.ninjagod.com', 'http://www.matchendirect.fr', 'http://www.daserste.de', 'http://www.icdrama.se', 'http://www.elperiodico.com', 'http://www.dbs.com.sg', 'http://www.couchsurfing.com', 'http://www.hesgoal.com', 'http://www.novate.ru', 'http://www.innervoice.ai', 'http://www.oxu.az', 'http://www.rutracker.net', 'http://www.onlinelife.club', 'http://www.videoblocks.com', 'http://www.showroomprive.com', 'http://www.mega.co.nz', 'http://www.one.com', 'http://www.skyrock.com', 'http://www.plos.org', 'http://www.exist.ru', 'http://www.elegantthemes.com', 'http://www.gasengi.com', 'http://www.xfcfinancial.online', 'http://www.tomtop.com', 'http://www.howtosimplified.com', 'http://www.invisionapp.com', 'http://www.forexfactory.com', 'http://www.kufar.by', 'http://www.javpop.com', 'http://www.futurenet.club', 'http://www.myasiantv.se', 'http://www.romsmania.cc', 'http://www.walkerland.com.tw', 'http://www.dogdrip.net', 'http://www.ogone.com', 'http://www.medialeaks.ru', 'http://www.filmesonlinehd7.cc', 'http://www.europixhd.com', 'http://www.mega-mult.ru', 'http://www.bdnews24.com', 'http://www.cna.com.tw', 'http://www.topuniversities.com', 'http://www.pornleech.is', 'http://www.bolalob.com', 'http://www.intesasanpaolo.com', 'http://www.coinmill.com', 'http://www.monsterindia.com', 'http://www.thefappening.pro', 'http://www.tatasky.com', 'http://www.underarmour.com', 'http://www.hossatin-tranced.com', 'http://www.rgo.ru', 'http://www.eplsite.biz', 'http://www.aarth.net', 'http://www.pravda.com.ua', 'http://www.biomedcentral.com', 'http://www.naukrigulf.com', 'http://www.kisscartoon.ac', 'http://www.tcs.com', 'http://www.unblockall.org', 'http://www.pandia.ru', 'http://www.dzone.com', 'http://www.wowkorea.jp', 'http://www.pornoamateurvip.com', 'http://www.nab.com.au', 'http://www.klook.com', 'http://www.extendoffice.com', 'http://www.povarenok.ru', 'http://www.freedoge.co.in', 'http://www.keywordtool.io', 'http://www.apost.com', 'http://www.fucktonite.com', 'http://www.monstercrawler.com', 'http://www.subdivx.com', 'http://www.yourvideofile.org', 'http://www.mymoviesda.com', 'http://www.newsit.gr', 'http://www.ctc.ru', 'http://www.apa.org', 'http://www.ceesty.com', 'http://www.mangahere.cc', 'http://www.newtabtvsearch.com', 'http://www.stopgame.ru', 'http://www.dubizzle.com', 'http://www.dizimag6.co', 'http://www.tripsavvy.com', 'http://www.sport1.de', 'http://www.gmx.com', 'http://www.porngem.com', 'http://www.zoosk.com', 'http://www.wizzair.com', 'http://www.debian.org', 'http://www.hattrick.org', 'http://www.up-4.net', 'http://www.fast-torrent.ru', 'http://www.alaraby.co.uk', 'http://www.alwqfajh.com', 'http://www.animenewsnetwork.com', 'http://www.jmty.jp', 'http://www.liverpool.com.mx', 'http://www.nnu.ng', 'http://www.sony.jp', 'http://www.usda.gov', 'http://www.simfileshare.net', 'http://www.kizi.com', 'http://www.fstsrv.com', 'http://www.trustedreviews.com', 'http://www.bom.gov.au', 'http://www.seloger.com', 'http://www.ipornovideos.xxx', 'http://www.tvtime.com', 'http://www.aksalser.com', 'http://www.recommendedforyou.xyz', 'http://www.yurk.com', 'http://www.arizona.edu', 'http://www.computerhoy.com', 'http://www.unimelb.edu.au', 'http://www.yande.re', 'http://www.sap.com', 'http://www.the-cinema.club', 'http://www.thor-hammer.pro', 'http://www.real.de', 'http://www.dba.dk', 'http://www.unibet.com', 'http://www.panzoid.com', 'http://www.iop.org', 'http://www.babyblog.ru', 'http://www.landsend.com', 'http://www.merrjep.com', 'http://www.gamestorrent.co', 'http://www.olx.uz', 'http://www.downloadha.com', 'http://www.karapaia.com', 'http://www.bb.com.br', 'http://www.ubuntuforums.org', 'http://www.journaldesfemmes.fr', 'http://www.zxpevjccjb.com', 'http://www.flibusta.is', 'http://www.marktplaats.nl', 'http://www.strawberrynet.com', 'http://www.chess24.com', 'http://www.teletalk.com.bd', 'http://www.azbyka.ru', 'http://www.gnula.se', 'http://www.yahoo.net', 'http://www.gsul.me', 'http://www.pokemonshowdown.com', 'http://www.incruit.com', 'http://www.vgtimes.ru', 'http://www.toexten.com', 'http://www.imslp.org', 'http://www.leroymerlin.fr', 'http://www.getresponse.com', 'http://www.williams-sonoma.com', 'http://www.dalfak.com', 'http://www.oantagonista.com', 'http://www.sharemods.com', 'http://www.ebrun.com', 'http://www.upsssc.gov.in', 'http://www.watchasian.co', 'http://www.interpals.net', 'http://www.alibaba.ir', 'http://www.doklad.ru', 'http://www.gmboxx.com', 'http://www.washingtonexaminer.com', 'http://www.computerbase.de', 'http://www.biji.co', 'http://www.hawaii.edu', 'http://www.barclays.co.uk', 'http://www.svpressa.ru', 'http://www.sportskeeda.com', 'http://www.intorrents.biz', 'http://www.com-repair-windows.live', 'http://www.alphashoppers.co', 'http://www.elsiglodetorreon.com.mx', 'http://www.elixx.me', 'http://www.rarbggo.org', 'http://www.androidfilehost.com', 'http://www.giallozafferano.it', 'http://www.onlinetrade.ru', 'http://www.cobrafilm.club', 'http://www.megdexchange.com', 'http://www.carmax.com', 'http://www.walkerplus.com', 'http://www.copart.com', 'http://www.getlnk7.com', 'http://www.lghtds.net', 'http://www.etryi.pro', 'http://www.textnow.com', 'http://www.animesvision.org', 'http://www.pravda.sk', 'http://www.watchcartoonsonline.la', 'http://www.maxpreps.com', 'http://www.radikal.ru', 'http://www.91mobiles.com', 'http://www.172mov.com', 'http://www.erail.in', 'http://www.vkmag.com', 'http://www.tripadvisor.fr', 'http://www.pictame.com', 'http://www.my-shop.ru', 'http://www.carsales.com.au', 'http://www.fmscout.com', 'http://www.mailchi.mp', 'http://www.flashscore.pl', 'http://www.dafiti.com.br', 'http://www.gaadiwaadi.com', 'http://www.gamerevolution.com', 'http://www.irancell.ir', 'http://www.healthcare.gov', 'http://www.thoughtcatalog.com', 'http://www.otakustream.tv', 'http://www.nielit.gov.in', 'http://www.theglobeandmail.com', 'http://www.lyon.kim', 'http://www.kolesa.kz', 'http://www.vov.vn', 'http://www.space.com', 'http://www.bigl.ua', 'http://www.anandabazar.com', 'http://www.onedio.ru', 'http://www.jetfilmizle.info', 'http://www.nowtv.com', 'http://www.skrill.com', 'http://www.lesnumeriques.com', 'http://www.iha.com.tr', 'http://www.seriesfilmestorrent.com', 'http://www.meusresultados.com', 'http://www.pagseguro.uol.com.br', 'http://www.channel4.com', 'http://www.glotchat.click', 'http://www.newsbomb.gr', 'http://www.comedyportal.net', 'http://www.dicionarioinformal.com.br', 'http://www.extra.com.br', 'http://www.lmv.io', 'http://www.goodhousekeeping.com', 'http://www.guru99.com', 'http://www.telecharger-youtube-mp3.com', 'http://www.aruba.it', 'http://www.virustotal.com', 'http://www.unwire.hk', 'http://www.pogo.com', 'http://www.yyets.com', 'http://www.jainists.us', 'http://www.aircanada.com', 'http://www.businessdictionary.com', 'http://www.uwaterloo.ca', 'http://www.insightsonindia.com', 'http://www.hitc.com', 'http://www.rtgfh.pro', 'http://www.scopus.com', 'http://www.jeffreestarcosmetics.com', 'http://www.adsupplyads.net', 'http://www.iconfinder.com', 'http://www.minecraft-inside.ru', 'http://www.launchpage.org', 'http://www.siemens.com', 'http://www.t-nation.com', 'http://www.germany.ru', 'http://www.24.hu', 'http://www.online-red.com', 'http://www.techadvisor.co.uk', 'http://www.voat.co', 'http://www.smotriserial.me', 'http://www.fdj.fr', 'http://www.duniagames.co.id', 'http://www.mathway.com', 'http://www.ilcorsaronero.ch', 'http://www.destinythegame.com', 'http://www.va.gov', 'http://www.poki.com', 'http://www.guatevision.com', 'http://www.thewhizproducts.com', 'http://www.uh.edu', 'http://www.homedepot.ca', 'http://www.levi.com', 'http://www.taqat.sa', 'http://www.guru3d.com', 'http://www.metal-archives.com', 'http://www.e-shop.gr', 'http://www.jorudan.co.jp', 'http://www.tgrthaber.com.tr', 'http://www.casadellibro.com', 'http://www.acbar.org', 'http://www.oregonstate.edu', 'http://www.pivigames.blog', 'http://www.kinokong2.com', 'http://www.lingualeo.com', 'http://www.ppt-online.org', 'http://www.wplay.co', 'http://www.futbollibre.online', 'http://www.livechatinc.com', 'http://www.nettruyen.com', 'http://www.myspace.com', 'http://www.cxfinancial.online', 'http://www.pokemondb.net', 'http://www.usajobs.gov', 'http://www.ameritrade.com', 'http://www.tdbank.com', 'http://www.modamania.es', 'http://www.eljur.ru', 'http://www.ilgeniodellostreaming.in', 'http://www.katadata.co.id', 'http://www.stoagergu.com', 'http://www.etrade.com', 'http://www.ewaybillgst.gov.in', 'http://www.tatar.ru', 'http://www.ancensored.com', 'http://www.ncsu.edu', 'http://www.tn.gov.in', 'http://www.1993s.top', 'http://www.studopedia.su', 'http://www.socialhub.online', 'http://www.pandao.ru', 'http://www.sendo.vn', 'http://www.yts-subtitles.com', 'http://www.myfantasyleague.com', 'http://www.kissasian.es', 'http://www.censor.net.ua', 'http://www.cgpersia.com', 'http://www.liveam.tv', 'http://www.touristtube.com', 'http://www.camelcamelcamel.com', 'http://www.zhibo.tv', 'http://www.jagonews24.com', 'http://www.eventim.de', 'http://www.hdron.tv', 'http://www.homemoviestube.com', 'http://www.furusato-tax.jp', 'http://www.purple6401.com', 'http://www.nzherald.co.nz', 'http://www.m-torrent.org', 'http://www.easycounter.com', 'http://www.alfabank.ru', 'http://www.wipro.com', 'http://www.lesiteinfo.com', 'http://www.7ya.ru', 'http://www.gsmhosting.com', 'http://www.corsair.com', 'http://www.av4.xyz', 'http://www.pullandbear.com', 'http://www.mapsofindia.com', 'http://www.btpian.com', 'http://www.redalyc.org', 'http://www.time.mk', 'http://www.aemet.es', 'http://www.westpac.com.au', 'http://www.nnmclub.to', 'http://www.stardewvalleywiki.com', 'http://www.videotrek.co', 'http://www.snapchat.com', 'http://www.oob009.site', 'http://www.cpmlink.net', 'http://www.corneey.com', 'http://www.sony.com', 'http://www.fontsquirrel.com', 'http://www.audioknigi.club', 'http://www.rizopoulospost.com', 'http://www.my-search.com', 'http://www.thebalancesmb.com', 'http://www.rae.es', 'http://www.1xbetua.mobi', 'http://www.hotwire.com', 'http://www.manychat.com', 'http://www.get-a-fuck-tonight.com', 'http://www.naszemiasto.pl', 'http://www.sahadan.com', 'http://www.usmagazine.com', 'http://www.ovagames.com', 'http://www.filmovmir.online', 'http://www.teatroporno.com', 'http://www.bbt.com', 'http://www.text.ru', 'http://www.dbr.ee', 'http://www.alabout.com', 'http://www.njuskalo.hr', 'http://www.pluska.sk', 'http://www.warframe.market', 'http://www.freetvonline.xyz', 'http://www.mbc.net', 'http://www.pooq.co.kr', 'http://www.digistyle.com', 'http://www.fastermirror.club', 'http://www.vogue.com', 'http://www.chatango.com', 'http://www.docplayer.ru', 'http://www.nseindia.com', 'http://www.akb48matomemory.com', 'http://www.transfermarkt.de', 'http://www.grantorrent.net', 'http://www.ascii.jp', 'http://www.heavymetalmachines.com', 'http://www.android30t.com', 'http://www.pornolomka.mobi', 'http://www.mzol7lbm.com', 'http://www.macworld.co.uk', 'http://www.zoombangla.com', 'http://www.tamildbox.biz', 'http://www.coingecko.com', 'http://www.sciencedaily.com', 'http://www.sydney.edu.au', 'http://www.flashback.org', 'http://www.dfpan.com', 'http://www.forebet.com', 'http://www.ivoox.com', 'http://www.moviesearchcenter.com', 'http://www.mongodb.com', 'http://www.criteo.com', 'http://www.timewarnercable.com', 'http://www.foxsportsgo.com', 'http://www.iwanttodeliver.com', 'http://www.mnrate.com', 'http://www.vktarget.ru', 'http://www.wallpaperswide.com', 'http://www.bonship-edules.com', 'http://www.lumenlearning.com', 'http://www.nme.com', 'http://www.legendas.tv', 'http://www.adbooth.com', 'http://www.timesofoman.com', 'http://www.soccerstand.com', 'http://www.spectrum.com', 'http://www.shop-pro.jp', 'http://www.pbskids.org', 'http://www.skynewsarabia.com', 'http://www.flightsearchapp.com', 'http://www.wowprogress.com', 'http://www.retty.me', 'http://www.kbx1sth37s.com', 'http://www.autobip.com', 'http://www.thestar.com.my', 'http://www.citruspay.com', 'http://www.surviv.io', 'http://www.win-rar.com', 'http://www.streamja.com', 'http://www.ani24zoa.com', 'http://www.9ku.com', 'http://www.atlasobscura.com', 'http://www.mediamarkt.es', 'http://www.myshows.me', 'http://www.techopedia.com', 'http://www.gamepress.gg', 'http://www.solarmovie.one', 'http://www.whatismyipaddress.com', 'http://www.cloudconvert.com', 'http://www.easysport.tv', 'http://www.fluentu.com', 'http://www.saramin.co.kr', 'http://www.accorhotels.com', 'http://www.layarkaca21.vip', 'http://www.sciencing.com', 'http://www.z2i.com', 'http://www.naurok.com.ua', 'http://www.directvnow.com', 'http://www.almrsal.com', 'http://www.vagalume.com.br', 'http://www.tradera.com', 'http://www.postype.com', 'http://www.senscritique.com', 'http://www.barlamane.com', 'http://www.takhfifan.com', 'http://www.discordbots.org', 'http://www.tuko.co.ke', 'http://www.residentadvisor.net', 'http://www.elgoles.me', 'http://www.coinpot.co', 'http://www.99designs.com', 'http://www.babla.ru', 'http://www.adam4adam.com', 'http://www.e-estekhdam.com', 'http://www.kino-fs.ucoz.net', 'http://www.sketchfab.com', 'http://www.aarp.org', 'http://www.heureka.sk', 'http://www.adguard.com', 'http://www.clicktripz.com', 'http://www.wallhaven.cc', 'http://www.cheezburger.com', 'http://www.tubidy.mobi', 'http://www.kickass2.cc', 'http://www.todocoleccion.net', 'http://www.rightel.ir', 'http://www.amadeus.com', 'http://www.edupage.org', 'http://www.muzlo.me', 'http://www.movistar.es', 'http://www.uiowa.edu', 'http://www.bonprix.ru', 'http://www.sportingnews.com', 'http://www.bigbasket.com', 'http://www.mbank.pl', 'http://www.seesaawiki.jp', 'http://www.jagranjosh.com', 'http://www.peru21.pe', 'http://www.tving.com', 'http://www.wa.gov', 'http://www.navitime.co.jp', 'http://www.shufoo.net', 'http://www.centrum24.pl', 'http://www.batepapo.uol.com.br', 'http://www.royalmail.com', 'http://www.livefootballol.me', 'http://www.16personalities.com', 'http://www.consubscribe.com', 'http://www.airtable.com', 'http://www.tamindir.com', 'http://www.ppshk.com', 'http://www.significados.com', 'http://www.mytoys.de', 'http://www.planningcenteronline.com', 'http://www.bibme.org', 'http://www.2nafare.com', 'http://www.zigwheels.com', 'http://www.winzip.com', 'http://www.picclick.com', 'http://www.skyscrapercity.com', 'http://www.bancobrasil.com.br', 'http://www.hitfile.net', 'http://www.kerala.gov.in', 'http://www.acesso.uol.com.br', 'http://www.shaadi.com', 'http://www.baseball-reference.com', 'http://www.gouv.qc.ca', 'http://www.otodom.pl', 'http://www.premiumtimesng.com', 'http://www.dailyhunt.in', 'http://www.diynetwork.com', 'http://www.minijuegos.com', 'http://www.prostoporno.vip', 'http://www.bitchute.com', 'http://www.img599.net', 'http://www.myplaycity.com', 'http://www.git-scm.com', 'http://www.t7479e4d.com', 'http://www.kseries.co', 'http://www.mkvcage.ws', 'http://www.bcxronvqkwe.com', 'http://www.yoursoccerdose.com', 'http://www.press-pulse.com', 'http://www.sportcategory.com', 'http://www.jofogas.hu', 'http://www.boohoo.com', 'http://www.mindmeister.com', 'http://www.ebalovo.info', 'http://www.mp3-muzyka.ru', 'http://www.trafficshop.com', 'http://www.sierratradingpost.com', 'http://www.itbazar.com', 'http://www.ktla.com', 'http://www.7days.ru', 'http://www.zhuixinfan.com', 'http://www.topwar.ru', 'http://www.oloadcdn.net', 'http://www.wowma.jp', 'http://www.maximonline.ru', 'http://www.japan-guide.com', 'http://www.descargatelocorp.com', 'http://www.sellfy.com', 'http://www.dndbeyond.com', 'http://www.akairan.com', 'http://www.angular.io', 'http://www.fubo.tv', 'http://www.gomovies.sc', 'http://www.practo.com', 'http://www.moviesbaba.cc', 'http://www.nespresso.com', 'http://www.french-stream.co', 'http://www.ipcorner.io', 'http://www.food123.com.tw', 'http://www.al-marsd.com', 'http://www.mallza.net', 'http://www.sanspo.com', 'http://www.sacbee.com', 'http://www.giveawayoftheday.com', 'http://www.gravatar.com', 'http://www.kidstaff.com.ua', 'http://www.keepo.me', 'http://www.centurylink.com', 'http://www.audacityteam.org', 'http://www.seosprint.net', 'http://www.mediametrics.ru', 'http://www.b92.net', 'http://www.lelivros.love', 'http://www.pudelek.pl', 'http://www.brilliant.org', 'http://www.arrlnk.com', 'http://www.telekom.de', 'http://www.pagesix.com', 'http://www.picodi.com', 'http://www.pornflip.com', 'http://www.liberation.fr', 'http://www.big.az', 'http://www.cilimao.me', 'http://www.unesco.org', 'http://www.lexpress.fr', 'http://www.animesonehd.net', 'http://www.korzik.net', 'http://www.torrentov.net', 'http://www.latercera.com', 'http://www.fontspace.com', 'http://www.echoroukonline.com', 'http://www.wholefoodsmarket.com', 'http://www.itbpolice.nic.in', 'http://www.hindawi.com', 'http://www.reklama5.mk', 'http://www.sbs.co.kr', 'http://www.iefimerida.gr', 'http://www.samlib.ru', 'http://www.pcloud.com', 'http://www.multoigri.ru', 'http://www.aternos.org', 'http://www.carwale.com', 'http://www.litnet.com', 'http://www.rst.ua', 'http://www.openedition.org', 'http://www.magicbricks.com', 'http://www.skylinewebcams.com', 'http://www.manchester.ac.uk', 'http://www.realtor.ca', 'http://www.goodgamestudios.com', 'http://www.rakuten-sec.co.jp', 'http://www.2dehands.be', 'http://www.transfermarkt.com', 'http://www.opendns.com', 'http://www.native-instruments.com', 'http://www.interfax.ru', 'http://www.kinoafisha.info', 'http://www.mmafighting.com', 'http://www.gdzputina.info', 'http://www.pbebank.com', 'http://www.avon.ru', 'http://www.freemake.com', 'http://www.adflyk.com', 'http://www.rhymezone.com', 'http://www.stream-direct.co', 'http://www.gossiplankanews.com', 'http://www.matomeantena.com', 'http://www.licindia.in', 'http://www.ura.news', 'http://www.colatour.com.tw', 'http://www.secretsdujeu.com', 'http://www.sigmalive.com', 'http://www.klikdokter.com', 'http://www.denofgeek.com', 'http://www.ti.com', 'http://www.animemovil.com', 'http://www.tkec.com.tw', 'http://www.anandtech.com', 'http://www.uwo.ca', 'http://www.ikea.cn', 'http://www.payscale.com', 'http://www.repelisplus.tv', 'http://www.khaleejtimes.com', 'http://www.adorama.com', 'http://www.upbasiceduboard.gov.in', 'http://www.setlist.fm', 'http://www.blogsky.com', 'http://www.altsantiri.gr', 'http://www.easyfileconvert.com', 'http://www.thegatewaypundit.com', 'http://www.trovaprezzi.it', 'http://www.tripadvisor.ca', 'http://www.saraiva.com.br', 'http://www.movs4u.tv', 'http://www.osxdaily.com', 'http://www.alleng.org', 'http://www.bookingbuddy.com', 'http://www.hostelworld.com', 'http://www.altema.jp', 'http://www.greenhouse.io', 'http://www.steamgifts.com', 'http://www.androidpolice.com', 'http://www.telstra.com.au', 'http://www.tcgplayer.com', 'http://www.newtab.club', 'http://www.ciliba.biz', 'http://www.canon.com', 'http://www.9to5mac.com', 'http://www.bollywoodshaadis.com', 'http://www.centrum.cz', 'http://www.mabanque.bnpparibas', 'http://www.brother.com', 'http://www.irishtimes.com', 'http://www.pianyuan.net', 'http://www.elitedaily.com', 'http://www.theage.com.au', 'http://www.internet-gazeta.com', 'http://www.westernunion.com', 'http://www.postimees.ee', 'http://www.apartmenttherapy.com', 'http://www.rufus.ie', 'http://www.mamibuy.com.tw', 'http://www.ssl2anyone5.com', 'http://www.arbeitsagentur.de', 'http://www.newpost.gr', 'http://www.xvideos10.blog.br', 'http://www.kprofiles.com', 'http://www.adultdvdempire.com', 'http://www.ixl.com', 'http://www.wangpan007.com', 'http://www.jawapos.com', 'http://www.escondeme.com', 'http://www.elblog.com', 'http://www.thecut.com', 'http://www.trakt.tv', 'http://www.melon.com', 'http://www.adafruit.com', 'http://www.bfads.net', 'http://www.900igr.net', 'http://www.straitstimes.com', 'http://www.otefauks.link', 'http://www.modthesims.info', 'http://www.ntv.com.tr', 'http://www.news-front.info', 'http://www.apurogol.net', 'http://www.dnb.no', 'http://www.koha.net', 'http://www.senego.com', 'http://www.mvideoporno.xxx', 'http://www.jarir.com', 'http://www.hsreplay.net', 'http://www.ntvbd.com', 'http://www.wordcounter.net', 'http://www.basecamp.com', 'http://www.1377x.to', 'http://www.fullmatchesandshows.com', 'http://www.caranddriver.com', 'http://www.diamond.jp', 'http://www.society6.com', 'http://www.cointelegraph.com', 'http://www.adultmult.tv', 'http://www.granbluefantasy.jp', 'http://www.domain.com.au', 'http://www.mp4moviez.trade', 'http://www.olx.bg', 'http://www.pornhdvideos.net', 'http://www.thegay.com', 'http://www.surveyrouter.com', 'http://www.carid.com', 'http://www.nurxxx.mobi', 'http://www.watchsport.fun', 'http://www.mercadolibre.com.pe', 'http://www.creativebloq.com', 'http://www.fwbhvrpiunlzyh.com', 'http://www.converto.io', 'http://www.poetryfoundation.org', 'http://www.adp-checker.ru', 'http://www.ubnt.com', 'http://www.windows.net', 'http://www.almstba.tv', 'http://www.cineaste.co.kr', 'http://www.rosserial.net', 'http://www.movcr.cc', 'http://www.b-ok.org', 'http://www.trueachievements.com', 'http://www.goldesel.to', 'http://www.webmasterplan.com', 'http://www.svoboda.org', 'http://www.vajehyab.com', 'http://www.lektsii.org', 'http://www.doktersehat.com', 'http://www.radiotimes.com', 'http://www.ifttt.com', 'http://www.twinkl.co.uk', 'http://www.ebaumsworld.com', 'http://www.routedurhum.com', 'http://www.seedr.cc', 'http://www.tn.com.ar', 'http://www.hardware.fr', 'http://www.scitation.org', 'http://www.usembassy.gov', 'http://www.zalando.fr', 'http://www.btbtdy.net', 'http://www.weatherblink.com', 'http://www.101.ru', 'http://www.themuse.com', 'http://www.cosme.net', 'http://www.cointiply.com', 'http://www.preonesetro.com', 'http://www.rtings.com', 'http://www.careerbuilder.com', 'http://www.ionos.com', 'http://www.liberoquotidiano.it', 'http://www.bittorrent.com', 'http://www.france24.com', 'http://www.au.com', 'http://www.mumyazh.com', 'http://www.shabdkosh.com', 'http://www.myrecipes.com', 'http://www.ntv.co.jp', 'http://www.nj.com', 'http://www.bdjobs.com', 'http://www.linktr.ee', 'http://www.chapintv.com', 'http://www.ncaa.com', 'http://www.koraextra.com', 'http://www.postila.ru', 'http://www.turkishairlines.com', 'http://www.mercola.com', 'http://www.dealnews.com', 'http://www.pokemon.com', 'http://www.fashionnova.com', 'http://www.starhit.ru', 'http://www.mgronline.com', 'http://www.newasiantv.ch', 'http://www.ppt4web.ru', 'http://www.tvnow.de', 'http://www.jaleco.com', 'http://www.namemc.com', 'http://www.gosearchresults.com', 'http://www.harpersbazaar.com', 'http://www.kmart.com', 'http://www.mrpiracy.xyz', 'http://www.ganol.st', 'http://www.whirlpool.net.au', 'http://www.adhoc2.net', 'http://www.descomplica.com.br', 'http://www.countryliving.com', 'http://www.powerthesaurus.org', 'http://www.barneys.com', 'http://www.axs.com', 'http://www.sulekha.com', 'http://www.hdstreams.club', 'http://www.dailydot.com', 'http://www.livemint.com', 'http://www.sfu.ca', 'http://www.hosyusokuhou.jp', 'http://www.neopets.com', 'http://www.name.com', 'http://www.worldstaruncut.com', 'http://www.snahp.it', 'http://www.moviesanywhere.com', 'http://www.gtmetrix.com', 'http://www.capterra.com', 'http://www.wdr.de', 'http://www.fernsehserien.de', 'http://www.onlinesoccermanager.com', 'http://www.torrentmegafilmes.com', 'http://www.activehosted.com', 'http://www.mmolift.com', 'http://www.haraj.com.sa', 'http://www.enamad.ir', 'http://www.unl.edu', 'http://www.adskeeper.co.uk', 'http://www.unlomblam.pro', 'http://www.femo.com', 'http://www.ua-football.com', 'http://www.wustl.edu', 'http://www.webcric.com', 'http://www.tvbersama.com', 'http://www.c-sharpcorner.com', 'http://www.fantasynamegenerators.com', 'http://www.toyota.com', 'http://www.radiorage.com', 'http://www.mobilism.org', 'http://www.abc7news.com', 'http://www.autoscout24.it', 'http://www.viraltungsten.com', 'http://www.edf.fr', 'http://www.elo7.com.br', 'http://www.jin115.com', 'http://www.gamovideo.com', 'http://www.news247.gr', 'http://www.themoviedb.org', 'http://www.thepiratebay.wf', 'http://www.libguides.com', 'http://www.animeshow.tv', 'http://www.gotomeeting.com', 'http://www.sexpennyauctions.com', 'http://www.libreoffice.org', 'http://www.tamildhool.com', 'http://www.nus.edu.sg', 'http://www.asda.com', 'http://www.just-eat.co.uk', 'http://www.project-free-tv.ag', 'http://www.e1.ru', 'http://www.bonus365.site', 'http://www.harborfreight.com', 'http://www.socialnewpagessearch.com', 'http://www.me.me', 'http://www.xn--41a.ws', 'http://www.softonic.com.br', 'http://www.linkshub.ws', 'http://www.newspoint.in', 'http://www.volbysr.sk', 'http://www.saavn.com', 'http://www.formulatv.com', 'http://www.popbela.com', 'http://www.imvu.com', 'http://www.softportal.com', 'http://www.movizland.online', 'http://www.argumenti.ru', 'http://www.cuevana3.co', 'http://www.e-monsite.com', 'http://www.gigporno.xxx', 'http://www.star.com.tr', 'http://www.vinted.fr', 'http://www.notebooksbilliger.de', 'http://www.extraimage.net', 'http://www.jbzdy.pl', 'http://www.zone-telechargement.world', 'http://www.libidgel.net', 'http://www.kidshealth.org', 'http://www.100xyev.com', 'http://www.slidescarnival.com', 'http://www.hani.co.kr', 'http://www.topg.org', 'http://www.masutabe.info', 'http://www.faberlic.com', 'http://www.auth0.com', 'http://www.speedvid.net', 'http://www.ngs.ru', 'http://www.lucidchart.com', 'http://www.viprow.net', 'http://www.smartsheet.com', 'http://www.colostate.edu', 'http://www.oliveboard.in', 'http://www.akurat.co', 'http://www.spiceworks.com', 'http://www.erome.com', 'http://www.aptg.com.tw', 'http://www.pelisplanet.com', 'http://www.jvzoo.com', 'http://www.carsensor.net', 'http://www.hepsibahis535.com', 'http://www.pastemagazine.com', 'http://www.samatika.com', 'http://www.hunker.com', 'http://www.hypixel.net', 'http://www.ipornogratis.xxx', 'http://www.core.ac.uk', 'http://www.watchanime.info', 'http://www.mturk.com', 'http://www.zoomg.ir', 'http://www.patrika.com', 'http://www.mercurynews.com', 'http://www.ucsb.edu', 'http://www.123movierulz.me', 'http://www.rojadirectaonlinetv.com', 'http://www.klm.com', 'http://www.doramasmp4.com', 'http://www.redflagdeals.com', 'http://www.rhvgmritmziwcm.net', 'http://www.ning.com', 'http://www.libsyn.com', 'http://www.manomano.fr', 'http://www.webex.com', 'http://www.ideegreen.it', 'http://www.manga-zip.net', 'http://www.bell.ca', 'http://www.factoftheday.net', 'http://www.toster.ru', 'http://www.noreste.net', 'http://www.emulator.games', 'http://www.givingassistant.org', 'http://www.qobuz.com', 'http://www.manchestereveningnews.co.uk', 'http://www.crichd.in', 'http://www.tfp.is', 'http://www.renderforest.com', 'http://www.ysl.com', 'http://www.hdo.to', 'http://www.highsnobiety.com', 'http://www.fboom.me', 'http://www.fzmovies.net', 'http://www.dexerto.com', 'http://www.travelask.ru', 'http://www.seneweb.com', 'http://www.rayadars.com', 'http://www.sanfoundry.com', 'http://www.majhinaukri.in', 'http://www.anilibria.tv', 'http://www.pusherism.com', 'http://www.edimdoma.ru', 'http://www.uptvs.com', 'http://www.torrent-tracker.co', 'http://www.voluumtrk.com', 'http://www.nta.ac.in', 'http://www.alza.sk', 'http://www.utdallas.edu', 'http://www.gamme.com.tw', 'http://www.nyc.gov', 'http://www.medicinenet.com', 'http://www.loft.com', 'http://www.phoenixads.net', 'http://www.skyeng.ru', 'http://www.phonearena.com', 'http://www.gsis.gr', 'http://www.ya.ru', 'http://www.hse.ru', 'http://www.vipbox.live', 'http://www.politexpert.net', 'http://www.ikman.lk', 'http://www.alrakoba.net', 'http://www.sexstories.com', 'http://www.tanea.gr', 'http://www.osvita.ua', 'http://www.progressive.com', 'http://www.goscr.co', 'http://www.xbuzzi.ir', 'http://www.ucl.ac.uk', 'http://www.openoffice.org', 'http://www.ondemandkorea.com', 'http://www.stepstone.de', 'http://www.cgpeers.com', 'http://www.bostonglobe.com', 'http://www.du.ac.in', 'http://www.shopee.sg', 'http://www.freemail.hu', 'http://www.grandars.ru', 'http://www.gofilmes.me', 'http://www.chinesean.com', 'http://www.airbnb.co.uk', 'http://www.lpages.co', 'http://www.smashbros.com', 'http://www.ooma.org', 'http://www.syri.net', 'http://www.startribune.com', 'http://www.cdon.se', 'http://www.docusign.com', 'http://www.auspost.com.au', 'http://www.wileyplus.com', 'http://www.sexporntube.us', 'http://www.deltarune.com', 'http://www.desbloqueador.org', 'http://www.beyazperde.com', 'http://www.indeed.co.uk', 'http://www.lacentrale.fr', 'http://www.jobvite.com', 'http://www.cnbtkitty.me', 'http://www.speedrun.com', 'http://www.tele2.ru', 'http://www.proprofs.com', 'http://www.wyylde.com', 'http://www.psl.pw', 'http://www.food52.com', 'http://www.metrolyrics.com', 'http://www.sigmaaldrich.com', 'http://www.gleglb.com', 'http://www.musiciansfriend.com', 'http://www.tableau.com', 'http://www.gledalica.com', 'http://www.riverisland.com', 'http://www.smule.com', 'http://www.tvrain.ru', 'http://www.justjared.com', 'http://www.codeforces.com', 'http://www.falafelandcaviar.com', 'http://www.holidaycheck.de', 'http://www.pahe.in', 'http://www.worldfree4u.lol', 'http://www.uio.no', 'http://www.footlocker.com', 'http://www.elibrary.ru', 'http://www.rarbgtor.org', 'http://www.bangumi.tv', 'http://www.tvzvezda.ru', 'http://www.ninemanga.com', 'http://www.pdfentity.co', 'http://www.watch-series.co', 'http://www.garena.tw', 'http://www.fili.cc', 'http://www.bajajfinserv.in', 'http://www.vagas.com.br', 'http://www.tv-tokyo.co.jp', 'http://www.wallpapercave.com', 'http://www.note.mu', 'http://www.huffingtonpost.jp', 'http://www.minecraft-mp.com', 'http://www.zety.com', 'http://www.1and1.com', 'http://www.iafd.com', 'http://www.prizegrab.com', 'http://www.yaporn.video', 'http://www.porno666.tv', 'http://www.eleseems-insector.com', 'http://www.material.io', 'http://www.watanserb.com', 'http://www.instamojo.com', 'http://www.bigcommerce.com', 'http://www.gotowebinar.com', 'http://www.talab.org', 'http://www.zum.com', 'http://www.gnu.org', 'http://www.vulcan.net.pl', 'http://www.ard.de', 'http://www.mirrorace.com', 'http://www.indiatyping.com', 'http://www.regions.com', 'http://www.rueconomics.ru', 'http://www.javjunkies.com', 'http://www.kemenag.go.id', 'http://www.time4education.com', 'http://www.cricsports.sc', 'http://www.sougouwiki.com', 'http://www.meshok.net', 'http://www.tommy.com', 'http://www.yifyhdtorrent.com', 'http://www.gtbank.com', 'http://www.thelastgame.ru', 'http://www.tradingeconomics.com', 'http://www.kitapyurdu.com', 'http://www.caradisiac.com', 'http://www.viks.tv', 'http://www.spirit.com', 'http://www.oload.download', 'http://www.tvsubtitles.net', 'http://www.newdelhitimes.com', 'http://www.special-alerts.com', 'http://www.nanowrimo.org', 'http://www.showmax.com', 'http://www.jalantikus.com', 'http://www.91wii.com', 'http://www.calc.ru', 'http://www.nanime.in', 'http://www.zalando.it', 'http://www.blick.ch', 'http://www.outlook.com', 'http://www.sky.de', 'http://www.sxkwor.space', 'http://www.fast-torrents.ru', 'http://www.iwank.tv', 'http://www.smithsonianmag.com', 'http://www.bathandbodyworks.com', 'http://www.tv-porinternet.com', 'http://www.sunnyplayer.com', 'http://www.webcrawler.com', 'http://www.borna.news', 'http://www.netvasco.com.br', 'http://www.jiji.com', 'http://www.sportschau.de', 'http://www.spring.io', 'http://www.paradoxplaza.com', 'http://www.club-k.net', 'http://www.gotquestions.org', 'http://www.sarkariresults.info', 'http://www.g1-globosaude.com', 'http://www.paisdelosjuegos.es', 'http://www.tripadvisor.es', 'http://www.ymovies.tv', 'http://www.wonderhowto.com', 'http://www.myopsports.org', 'http://www.oa.com', 'http://www.getlnk5.com', 'http://www.financenancy.com', 'http://www.buffalo.edu', 'http://www.futbolparatodos.net', 'http://www.grandxxi.com', 'http://www.ilmessaggero.it', 'http://www.iflscience.com', 'http://www.adaware.com', 'http://www.bizjournals.com', 'http://www.majorleaguegaming.com', 'http://www.wits.ac.za', 'http://www.bigfishgames.com', 'http://www.zapmetasearch.com', 'http://www.openloadmovies.net', 'http://www.bale.ai', 'http://www.directadvert.ru', 'http://www.mobilekomak.com', 'http://www.lidl-flyer.com', 'http://www.keywordblocks.com', 'http://www.homeaffairs.gov.au', 'http://www.baiscopelk.com', 'http://www.freshersworld.com', 'http://www.fox.com', 'http://www.studopedia.org', 'http://www.filepuma.com', 'http://www.stopmusic.net', 'http://www.gunbroker.com', 'http://www.shacknews.com', 'http://www.minecraftskins.com', 'http://www.toxicwap.com', 'http://www.demotywatory.pl', 'http://www.dailytvfix.com', 'http://www.bhg.com', 'http://www.cgpeers.to', 'http://www.loveread.ec', 'http://www.tube2012.com', 'http://www.wayfair.ca', 'http://www.123link.pro', 'http://www.eleconomista.es', 'http://www.boxrec.com', 'http://www.pomponik.pl', 'http://www.svscomics.com', 'http://www.uq.edu.au', 'http://www.skribbl.io', 'http://www.roku.com', 'http://www.giantbomb.com', 'http://www.euro.com.pl', 'http://www.abercrombie.com', 'http://www.usf.edu', 'http://www.cathaypacific.com', 'http://www.merylaural.info', 'http://www.aastocks.com', 'http://www.mediaite.com', 'http://www.unec.edu.az', 'http://www.nationwide.co.uk', 'http://www.verywellmind.com', 'http://www.kindgirls.com', 'http://www.msu.ru', 'http://www.manithan.com', 'http://www.pptcloud.ru', 'http://www.aftenposten.no', 'http://www.coinschedule.com', 'http://www.yealnk.com', 'http://www.kinoxa.net', 'http://www.niksalehi.com', 'http://www.libertaddigital.com', 'http://www.uptolink.xyz', 'http://www.f1livegp.net', 'http://www.onclickpulse.com', 'http://www.tv2.no', 'http://www.8live.com', 'http://www.mycdn.me', 'http://www.kobo.com', 'http://www.tweowhvrim.review', 'http://www.cuhk.edu.hk', 'http://www.superjob.ru', 'http://www.sbi.co.in', 'http://www.getlnk4.com', 'http://www.hotline.ua', 'http://www.123images.co', 'http://www.joyclub.de', 'http://www.220-volt.ru', 'http://www.uspto.gov', 'http://www.ionos.de', 'http://www.kataweb.it', 'http://www.zarinpal.com', 'http://www.fxnetworks.com', 'http://www.kartaslov.ru', 'http://www.freepdfconvert.com', 'http://www.santabanta.com', 'http://www.scambioetico.org', 'http://www.adevarul.ro', 'http://www.warthunder.com', 'http://www.iphones.ru', 'http://www.1be.biz', 'http://www.filmeserialeonline.org', 'http://www.bose.com', 'http://www.venmo.com', 'http://www.coinsbank.com', 'http://www.comdirect.de', 'http://www.kanoon.ir', 'http://www.muchohentai.com', 'http://www.opinionsample.com', 'http://www.c-and-a.com', 'http://www.123telugu.com', 'http://www.cliver.tv', 'http://www.ero-advertising.com', 'http://www.4game.com', 'http://www.a10.com', 'http://www.surveygizmo.com', 'http://www.you-porn.com', 'http://www.mrdoob.com', 'http://www.saat24.news', 'http://www.alodoctor.com', 'http://www.brainberries.co', 'http://www.multporn.net', 'http://www.thebay.com', 'http://www.businessinsider.com.pl', 'http://www.kaymopk.com', 'http://www.salon.com', 'http://www.mma-core.com', 'http://www.hvg.hu', 'http://www.neswangy.net', 'http://www.ddizim.com', 'http://www.clickjogos.com.br', 'http://www.moondash.co.in', 'http://www.aljaras.com', 'http://www.beru.ru', 'http://www.bibliocommons.com', 'http://www.pornexpert.net', 'http://www.telegraphindia.com', 'http://www.vix.com', 'http://www.mangapark.me', 'http://www.rsload.net', 'http://www.flipboard.com', 'http://www.atpworldtour.com', 'http://www.yooying.com', 'http://www.buyeasy.by', 'http://www.imagefapusercontent.com', 'http://www.vsetop.org', 'http://www.gbatemp.net', 'http://www.theblaze.com', 'http://www.smartserial.club', 'http://www.toreents.club', 'http://www.dostfilms.net', 'http://www.techrepublic.com', 'http://www.ejemplos.co', 'http://www.dokumen.tips', 'http://www.dramacool9.io', 'http://www.vercanalestv1.com', 'http://www.directhit.com', 'http://www.decider.com', 'http://www.cumhuriyet.com.tr', 'http://www.vvvdj.com', 'http://www.ilsole24ore.com', 'http://www.asia2tv.co', 'http://www.psonstrentie.info', 'http://www.boots.com', 'http://www.jobriya.in', 'http://www.spcs.me', 'http://www.sockshare.net', 'http://www.pushmobilenews.com', 'http://www.0123putlocker.com', 'http://www.goeuro.com', 'http://www.shafa.ua', 'http://www.sanet.st', 'http://www.moonbit.co.in', 'http://www.only-tv.org', 'http://www.fanpage.it', 'http://www.jdsports.co.uk', 'http://www.op.fi', 'http://www.coop-land.ru', 'http://www.byu.edu', 'http://www.vev.io', 'http://www.livescore.in', 'http://www.apornstories.com', 'http://www.prohbtd.com', 'http://www.thestudentroom.co.uk', 'http://www.wnd.com', 'http://www.skokka.com', 'http://www.passeportsante.net', 'http://www.xrivonet.info', 'http://www.streamonsports.com', 'http://www.rte.ie', 'http://www.opizo.me', 'http://www.clevernt.com', 'http://www.b17.ru', 'http://www.pronews.gr', 'http://www.slideplayer.es', 'http://www.redhat.com', 'http://www.somosmovies.com', 'http://www.sta.sh', 'http://www.vividseats.com', 'http://www.dawnnews.tv', 'http://www.wordhippo.com', 'http://www.cleartax.in', 'http://www.searchalgo.com', 'http://www.rtbs24.com', 'http://www.ubs.com', 'http://www.rlsnet.ru', 'http://www.bonprix.de', 'http://www.rarbg2018.org', 'http://www.empik.com', 'http://www.123movies.net', 'http://www.djangoproject.com', 'http://www.aqicn.org', 'http://www.newsbeast.gr', 'http://www.cognizant.com', 'http://www.ygosu.com', 'http://www.looper.com', 'http://www.justanswer.com', 'http://www.bokepxv.com', 'http://www.apteka.ru', 'http://www.financialexpress.com', 'http://www.gmu.edu', 'http://www.cmoney.tw', 'http://www.nickiswift.com', 'http://www.airbnb.ru', 'http://www.motor-talk.de', 'http://www.vtb.ru', 'http://www.military.com', 'http://www.texas.gov', 'http://www.sdpnoticias.com', 'http://www.tudocelular.com', 'http://www.drupal.org', 'http://www.vklass.se', 'http://www.cricket.com.au', 'http://www.osym.gov.tr', 'http://www.spletnik.ru', 'http://www.mumsnet.com', 'http://www.baby.ru', 'http://www.carrefour.es', 'http://www.joemonster.org', 'http://www.ilibrary.ru', 'http://www.moppy.jp', 'http://www.vidz7.com', 'http://www.click108.com.tw', 'http://www.malaysiakini.com', 'http://www.newsru.co.il', 'http://www.sibnet.ru', 'http://www.arenavision.in', 'http://www.dafapromo.com', 'http://www.clever.com', 'http://www.fanatics.com', 'http://www.health.com', 'http://www.turkcealtyazi.org', 'http://www.blesk.cz', 'http://www.studocu.com', 'http://www.wpengine.com', 'http://www.lhscan.net', 'http://www.cpubenchmark.net', 'http://www.shiksha.com', 'http://www.mixer-1ruplus.ru', 'http://www.pornomovies.com', 'http://www.buscape.com.br', 'http://www.eobot.com', 'http://www.hornbach.de', 'http://www.kmplayer.com', 'http://www.typingclub.com', 'http://www.591.com.tw', 'http://www.nullschool.net', 'http://www.niusnews.com', 'http://www.pekao24.pl', 'http://www.visioncloud.ga', 'http://www.multiup.org', 'http://www.reference.com', 'http://www.senasofiaplus.edu.co', 'http://www.rulit.me', 'http://www.sainsburys.co.uk', 'http://www.amung.us', 'http://www.vuejs.org', 'http://www.knaben.cc', 'http://www.handy-tab.com', 'http://www.stylecraze.com', 'http://www.fanwenwangzhan.com', 'http://www.enow.com', 'http://www.povwideo.cc', 'http://www.sncf.com', 'http://www.freesport.info', 'http://www.arablionz.online', 'http://www.telegra.ph', 'http://www.bludv.tv', 'http://www.xmovies8.pl', 'http://www.cyberciti.biz', 'http://www.tonkosti.ru', 'http://www.5-tv.ru', 'http://www.petsmart.com', 'http://www.cell.com', 'http://www.bitchoko.biz', 'http://www.sport.pl', 'http://www.queensu.ca', 'http://www.lawsivo.ru', 'http://www.rakuten-bank.co.jp', 'http://www.toptvshows.co', 'http://www.ku.edu', 'http://www.aftership.com', 'http://www.kbs.co.kr', 'http://www.extrastores.com', 'http://www.mirconnect.ru', 'http://www.loc.gov', 'http://www.experian.com', 'http://www.walmart.com.mx', 'http://www.xxxstreams.org', 'http://www.babbel.com', 'http://www.dartmouth.edu', 'http://www.tenor.com', 'http://www.national-lottery.co.uk', 'http://www.hilltopads.net', 'http://www.livehd7.com', 'http://www.fuegodevida.com', 'http://www.publico.es', 'http://www.expatriates.com', 'http://www.ashtgalat.com', 'http://www.dezeen.com', 'http://www.manuscriptcentral.com', 'http://www.fox.com.tr', 'http://www.yorku.ca', 'http://www.vuzlit.ru', 'http://www.sportsguru.in', 'http://www.signupgenius.com', 'http://www.uoregon.edu', 'http://www.futurelearn.com', 'http://www.treccani.it', 'http://www.lww.com', 'http://www.katespade.com', 'http://www.laiguana.tv', 'http://www.menshealth.com', 'http://www.optus.com.au', 'http://www.eurosport.com', 'http://www.daily.co.jp', 'http://www.menhdv.com', 'http://www.razlozhi.ru', 'http://www.animespirit.ru', 'http://www.technopoint.ru', 'http://www.declk.com', 'http://www.dzexams.com', 'http://www.uniregistry.com', 'http://www.ajel.sa', 'http://www.melodrama1.com', 'http://www.timesofmalta.com', 'http://www.epa.com.py', 'http://www.pagina12.com.ar', 'http://www.foodandwine.com', 'http://www.byjus.com', 'http://www.gamebanana.com', 'http://www.mrskin.com', 'http://www.voirfilms.ws', 'http://www.fasttorrent.su', 'http://www.arukereso.hu', 'http://www.techpowerup.com', 'http://www.songsmp3.org', 'http://www.centerblog.net', 'http://www.skelbiu.lt', 'http://www.adjara.com', 'http://www.newchic.com', 'http://www.skiddle.com', 'http://www.norwegian.com', 'http://www.resurrect.club', 'http://www.jiji.ng', 'http://www.english-films.com', 'http://www.metod-kopilka.ru', 'http://www.mediaad.org', 'http://www.decipherinc.com', 'http://www.elkhamis.com', 'http://www.yapo.cl', 'http://www.filebase.ws', 'http://www.999.md', 'http://www.admitad.com', 'http://www.top4top.net', 'http://www.clickatlas.com', 'http://www.freshnewsasia.com', 'http://www.pornsos.com', 'http://www.getyabrowser.net', 'http://www.anoboy.org', 'http://www.reddmn.com', 'http://www.railyatri.in', 'http://www.work.ua', 'http://www.audiojungle.net', 'http://www.ucanapply.com', 'http://www.myfonts.com', 'http://www.hku.hk', 'http://www.unieuro.it', 'http://www.pciconcursos.com.br', 'http://www.mp.gov.in', 'http://www.tenforums.com', 'http://www.kinoplanet.net', 'http://www.fsu.edu', 'http://www.hdfull.me', 'http://www.persiantools.com', 'http://www.pagesperso-orange.fr', 'http://www.deutschepost.de', 'http://www.myschoolapp.com', 'http://www.camgirlsowned.com', 'http://www.moonliteco.in', 'http://www.donedeal.ie', 'http://www.teamtreehouse.com', 'http://www.ddanzi.com', 'http://www.gsm.ir', 'http://www.policybazaar.com', 'http://www.xgap.tv', 'http://www.cartoonmad.com', 'http://www.bd-film.cc', 'http://www.synology.com', 'http://www.keeplinks.xyz', 'http://www.frandroid.com', 'http://www.rd.com', 'http://www.onelogin.com', 'http://www.zerochan.net', 'http://www.bobibanking.com', 'http://www.univie.ac.at', 'http://www.tampermonkey.net', 'http://www.agenciavix.com', 'http://www.intercambiosvirtuales.org', 'http://www.homepage-web.com', 'http://www.rtl.de', 'http://www.viator.com', 'http://www.bankinfoindia.com', 'http://www.tlauncher.org', 'http://www.gearslutz.com', 'http://www.ryerson.ca', 'http://www.desixnxx.net', 'http://www.mcdonalds.com', 'http://www.animeindo.video', 'http://www.cnrtl.fr', 'http://www.aconvert.com', 'http://www.nhlstream.xyz', 'http://www.ucsc.edu', 'http://www.mandatory.com', 'http://www.gsp.ro', 'http://www.studme.org', 'http://www.classiccars.com', 'http://www.doramakun.ru', 'http://www.prepinsta.com', 'http://www.nest.com', 'http://www.gopro.com', 'http://www.evrl.to', 'http://www.eleman.net', 'http://www.kaidee.com', 'http://www.flashx.co', 'http://www.film.ru', 'http://www.linguee.es', 'http://www.gigdnetwork.com', 'http://www.faceapp.com', 'http://www.qrlsx.com', 'http://www.ballzaa.com', 'http://www.pressroomvip.online', 'http://www.semana.com', 'http://www.tinymz.info', 'http://www.ftchinese.com', 'http://www.kwfinder.com', 'http://www.audiobookbay.nl', 'http://www.bsi.ir', 'http://www.yume551.com', 'http://www.kuleuven.be', 'http://www.channelionline.com', 'http://www.roguard.net', 'http://www.net.hr', 'http://www.speedvideo.net', 'http://www.aftabir.com', 'http://www.channelnewsasia.com', 'http://www.jpg2pdf.com', 'http://www.thesaltymarshmallow.com', 'http://www.tvklan.al', 'http://www.lefrecce.it', 'http://www.playoverwatch.com', 'http://www.markt.de', 'http://www.relativelyinteresting.com', 'http://www.manta.com', 'http://www.oanda.com', 'http://www.vashurok.ru', 'http://www.symbaloo.com', 'http://www.lalafo.az', 'http://www.pinflix.com', 'http://www.alfalfalfa.com', 'http://www.stan.com.au', 'http://www.youtubeto.com', 'http://www.coccoc.com', 'http://www.zme8o3l1c4.com', 'http://www.track24.ru', 'http://www.blogos.com', 'http://www.getformsonline.com', 'http://www.pedsovet.su', 'http://www.niazerooz.com', 'http://www.foxplay.com', 'http://www.oyunfor.com', 'http://www.cimri.com', 'http://www.boxof.porn', 'http://www.adultswim.com', 'http://www.helpiks.org', 'http://www.statcounter.com', 'http://www.orient-news.net', 'http://www.sportsnet.ca', 'http://www.sdsjweb.com', 'http://www.uga.edu', 'http://www.insight.co.kr', 'http://www.adfoc.us', 'http://www.prokazi.com', 'http://www.genk.vn', 'http://www.yts.vg', 'http://www.wifi.free.fr', 'http://www.rushporn.com', 'http://www.morhipo.com', 'http://www.cardekho.com', 'http://www.bfm.ru', 'http://www.howto-news.info', 'http://www.meritnation.com', 'http://www.eurostreaming.news', 'http://www.nios.ac.in', 'http://www.cricfree.live', 'http://www.al-madina.com', 'http://www.itesm.mx', 'http://www.toluna.com', 'http://www.petapixel.com', 'http://www.rbi.org.in', 'http://www.onlineradiobox.com', 'http://www.rarbgunblocked.org', 'http://www.hackerearth.com', 'http://www.thegameraccess.com', 'http://www.dailyuploads.net', 'http://www.exidecare.com', 'http://www.qconcursos.com', 'http://www.newsread.info', 'http://www.joann.com', 'http://www.redbus.in', 'http://www.fanatik.com.tr', 'http://www.showroom-live.com', 'http://www.faucethub.io', 'http://www.anime-sharing.com', 'http://www.ecnavi.jp', 'http://www.tonicmovies.com', 'http://www.finalfantasyxiv.com', 'http://www.decathlon.fr', 'http://www.lifebru.com', 'http://www.myanimesonline.net', 'http://www.westlandstorage.com', 'http://www.thefuncoolstuff.com', 'http://www.meteoblue.com', 'http://www.splice.com', 'http://www.pontofrio.com.br', 'http://www.siriusxm.com', 'http://www.baixedetudo.net', 'http://www.gaoqingkong.com', 'http://www.1dl.biz', 'http://www.ntu.edu.sg', 'http://www.hasznaltauto.hu', 'http://www.acestream.org', 'http://www.zopim.com', 'http://www.get-express-vpn.com', 'http://www.iitb.ac.in', 'http://www.fromhot.com', 'http://www.shemalez.com', 'http://www.alldatasheet.com', 'http://www.dfiles.eu', 'http://www.2ch.sc', 'http://www.jet.com', 'http://www.pracuj.pl', 'http://www.bancsabadell.com', 'http://www.alahlionline.com', 'http://www.warwick.ac.uk', 'http://www.beforward.jp', 'http://www.kimovil.com', 'http://www.buyma.com', 'http://www.viagogo.com', 'http://www.coversong.info', 'http://www.emol.com', 'http://www.steamcharts.com', 'http://www.derpibooru.org', 'http://www.tripadvisor.com.au', 'http://www.gamespark.jp', 'http://www.screwfix.com', 'http://www.hanfan.cc', 'http://www.sur.ly', 'http://www.worldmarket.com', 'http://www.streeteasy.com', 'http://www.ukessays.com', 'http://www.scholarships.gov.in', 'http://www.saplinglearning.com', 'http://www.lifehack.org', 'http://www.apowersoft.com', 'http://www.gtavicecity.ru', 'http://www.pronto.com', 'http://www.filmmodu.com', 'http://www.mappy.com', 'http://www.nokia.com', 'http://www.ladepeche.fr', 'http://www.mixporn24.com', 'http://www.education.fr', 'http://www.torrentfunk2.com', 'http://www.robinhood.com', 'http://www.filmshd.club', 'http://www.d20pfsrd.com', 'http://www.canarabank.in', 'http://www.kau.edu.sa', 'http://www.cic.fr', 'http://www.autotrader.ca', 'http://www.tinkercad.com', 'http://www.bmail.uol.com.br', 'http://www.tripadvisor.de', 'http://www.moondoge.co.in', 'http://www.cinemitas.com', 'http://www.tumarcador.xyz', 'http://www.geo.tv', 'http://www.iceimg.net', 'http://www.yamoney.ru', 'http://www.ap.org', 'http://www.mec.es', 'http://www.excelsior.com.mx', 'http://www.flightrising.com', 'http://www.zhangw8.com', 'http://www.moscatalogue.net', 'http://www.classdojo.com', 'http://www.ninjajournalist.com', 'http://www.steamdb.info', 'http://www.payeer.com', 'http://www.persee.fr', 'http://www.gosugamers.net', 'http://www.123movieshubz.com', 'http://www.turkanime.tv', 'http://www.goonj.pk', 'http://www.topshop.com', 'http://www.rec-tube.com', 'http://www.clubfactory.com', 'http://www.cracked-games.org', 'http://www.nascar.com', 'http://www.q8yat.com', 'http://www.kia.com', 'http://www.torrentfilmes.net', 'http://www.a2hosting.com', 'http://www.igromania.ru', 'http://www.tomtom.com', 'http://www.tasty.co', 'http://www.proza.ru', 'http://www.rockfile.co', 'http://www.cineplex.com', 'http://www.sportpesa.co.ke', 'http://www.jeevansathi.com', 'http://www.movie-blog.to', 'http://www.camvideos.tv', 'http://www.portaleducacao.com.br', 'http://www.pydata.org', 'http://www.cuisineaz.com', 'http://www.cncv.org.cn', 'http://www.ngacn.cc', 'http://www.draw.io', 'http://www.meinestadt.de', 'http://www.videodownloaderultimate.com', 'http://www.longdo.com', 'http://www.free-power-point-templates.com', 'http://www.sofifa.com', 'http://www.aleks.com', 'http://www.mvnrepository.com', 'http://www.si.edu', 'http://www.cathaybk.com.tw', 'http://www.blog.hu', 'http://www.chrono.gg', 'http://www.1xppwu.host', 'http://www.goud.ma', 'http://www.winbank.gr', 'http://www.baypirateproxy.org', 'http://www.politikus.ru', 'http://www.sponichi.co.jp', 'http://www.nuevoloquo.com', 'http://www.emory.edu', 'http://www.sudouest.fr', 'http://www.phimbathu.com', 'http://www.qaru.site', 'http://www.iol.pt', 'http://www.yoo7.com', 'http://www.noon.com', 'http://www.mysmartprice.com', 'http://www.moorddiner.co', 'http://www.ruanyifeng.com', 'http://www.marthastewart.com', 'http://www.kun.uz', 'http://www.aai.aero', 'http://www.huntington.com', 'http://www.vedantu.com', 'http://www.kookporn.com', 'http://www.musinsa.com', 'http://www.hottopic.com', 'http://www.riie.net', 'http://www.telecup.com', 'http://www.hirunews.lk', 'http://www.lastsecond.ir', 'http://www.hoofoot.com', 'http://www.securecloud-ec.com', 'http://www.onehallyu.com', 'http://www.jamiiforums.com', 'http://www.shabakaty.com', 'http://www.yousport.live', 'http://www.sonymobile.com', 'http://www.coolpc.com.tw', 'http://www.autoplius.lt', 'http://www.rpp.pe', 'http://www.gencat.cat', 'http://www.rustih.ru', 'http://www.civilica.com', 'http://www.tsumino.com', 'http://www.firmwarefile.com', 'http://www.tabletki.ua', 'http://www.centrum.sk', 'http://www.thebase.in', 'http://www.adturtle.biz', 'http://www.voirfims.ws', 'http://www.ettv.tv', 'http://www.walmart.com.br', 'http://www.safeurl.net', 'http://www.tufos.com.br', 'http://www.pornocarioca.com', 'http://www.blogmura.com', 'http://www.tori.fi', 'http://www.payback.de', 'http://www.netzwelt.de', 'http://www.ebay.at', 'http://www.bestfreetube.xxx', 'http://www.sdmoviespoint.club', 'http://www.online2pdf.com', 'http://www.dll-files.com', 'http://www.infoniac.ru', 'http://www.mrvideospornogratis.xxx', 'http://www.tatacliq.com', 'http://www.neu.edu', 'http://www.kinozal.tv', 'http://www.qoo10.com', 'http://www.tranny.one', 'http://www.persianv.com', 'http://www.gmaegames.pro', 'http://www.estacio.br', 'http://www.jobkorea.co.kr', 'http://www.bibostream.club', 'http://www.tasvirezendegi.com', 'http://www.northeastern.edu', 'http://www.thinkgeek.com', 'http://www.rusprofile.ru', 'http://www.gtorrent.net', 'http://www.consoupow.com', 'http://www.notepad-plus-plus.org', 'http://www.nahnews.org', 'http://www.pushedwebnews.com', 'http://www.doctr1ne.com', 'http://www.neliti.com', 'http://www.nba-stream.com', 'http://www.searchdconvertnow.com', 'http://www.ucoz.com', 'http://www.h83zvgrg29.com', 'http://www.marketingvici.com', 'http://www.moonbitcoin.cash', 'http://www.pniao.com', 'http://www.convertkit.com', 'http://www.rozblog.com', 'http://www.pidruchniki.com', 'http://www.arabsong.top', 'http://www.bwin.com', 'http://www.anzalweb.ir', 'http://www.eghtesadonline.com', 'http://www.myself-bbs.com', 'http://www.archlinux.org', 'http://www.bbb.org', 'http://www.naasongs.com', 'http://www.pepperfry.com', 'http://www.patheos.com', 'http://www.online-video-cutter.com', 'http://www.warez-bb.org', 'http://www.sinonim.org', 'http://www.pec.it', 'http://www.pythonteens.com', 'http://www.sxstube.com', 'http://www.healthunlocked.com', 'http://www.spokeo.com', 'http://www.doostihaa.com', 'http://www.forumfree.it', 'http://www.withoutworking.com', 'http://www.seword.com', 'http://www.trueid.net', 'http://www.backcountry.com', 'http://www.cairn.info', 'http://www.mediaexpert.pl', 'http://www.picclick.de', 'http://www.anysearchmanager.com', 'http://www.swappa.com', 'http://www.testberichte.de', 'http://www.zapmeta.co.in', 'http://www.novayagazeta.ru', 'http://www.hdkino.biz', 'http://www.aktualne.cz', 'http://www.minhavida.com.br', 'http://www.videa.hu', 'http://www.hululkitab.com', 'http://www.pojoksatu.id', 'http://www.povar.ru', 'http://www.statefarm.com', 'http://www.sinemalar.com', 'http://www.vans.com', 'http://www.kodi.tv', 'http://www.48idol.com', 'http://www.msj1.com', 'http://www.bankia.es', 'http://www.groupme.com', 'http://www.videospornogratisx.net', 'http://www.cntd.ru', 'http://www.hateblo.jp', 'http://www.jvpnews.com', 'http://www.tinyurl.com', 'http://www.zlsite.com', 'http://www.physicsforums.com', 'http://www.bankmillennium.pl', 'http://www.av.by', 'http://www.newzfeed.ru', 'http://www.airdroid.com', 'http://www.mako.co.il', 'http://www.vipleague.lc', 'http://www.xhcdn.com', 'http://www.ivfree.me', 'http://www.1-tube.ru', 'http://www.alyamanalaraby.com', 'http://www.cbr.com', 'http://www.interwetten.com', 'http://www.games-workshop.com', 'http://www.partis.si', 'http://www.360zhibo.com', 'http://www.horoscope.com', 'http://www.quidco.com', 'http://www.meteo.gr', 'http://www.hentai-chan.me', 'http://www.gjirafa.com', 'http://www.travelandleisure.com', 'http://www.faselhd.co', 'http://www.cdkeys.com', 'http://www.elliberal.com.ar', 'http://www.radiorecord.ru', 'http://www.indishare.me', 'http://www.applealmond.com', 'http://www.ufc.tv', 'http://www.anyanime.com', 'http://www.news24nepal.tv', 'http://www.costco.com.tw', 'http://www.2banh.vn', 'http://www.francaisfacile.com', 'http://www.iol.co.za', 'http://www.sprzedajemy.pl', 'http://www.paradoxwikis.com', 'http://www.managebac.com', 'http://www.hwnews.in', 'http://www.skladchik.com', 'http://www.uproxx.com', 'http://www.fenglish.ru', 'http://www.huffingtonpost.ca', 'http://www.accengage.net', 'http://www.zap2it.com', 'http://www.politros.com', 'http://www.hungama.com', 'http://www.softlay.net', 'http://www.superflix.net', 'http://www.experts-exchange.com', 'http://www.ek.ua', 'http://www.marvel.com', 'http://www.omicsonline.org', 'http://www.pearsonvue.com', 'http://www.quicklaunchsso.com', 'http://www.ajio.com', 'http://www.bradesco.com.br', 'http://www.cnnturk.com', 'http://www.booru.org', 'http://www.seriestreaming.site', 'http://www.hotcleaner.com', 'http://www.stranamam.ru', 'http://www.timesjobs.com', 'http://www.tamwin.com.eg', 'http://www.biography.com', 'http://www.raiffeisen.at', 'http://www.rei6ohka.com', 'http://www.translate.ru', 'http://www.georgetown.edu', 'http://www.bankhapoalim.co.il', 'http://www.komplett.se', 'http://www.turkishtv.ru', 'http://www.360tv.ru', 'http://www.ucm.es', 'http://www.vhlcentral.com', 'http://www.cardmarket.com', 'http://www.ancestry.co.uk', 'http://www.slideserve.com', 'http://www.epson.com', 'http://www.ecoledirecte.com', 'http://www.mercadopago.com', 'http://www.goodfon.ru', 'http://www.pubg.com', 'http://www.uva.nl', 'http://www.porofessor.gg', 'http://www.tejaratbank.ir', 'http://www.odatv.com', 'http://www.wallhere.com', 'http://www.baixarseriesmp4.org', 'http://www.mahadbtmahait.gov.in', 'http://www.pdftoword.com', 'http://www.jjang0u.com', 'http://www.yts.gs', 'http://www.jansatta.com', 'http://www.seattletimes.com', 'http://www.evo.com', 'http://www.f1news.ru', 'http://www.moviepilot.de', 'http://www.pacogames.com', 'http://www.yota.ru', 'http://www.mysurvey.com', 'http://www.essahraa.net', 'http://www.coches.net', 'http://www.windowsazure.com', 'http://www.conforama.fr', 'http://www.escort-advisor.com', 'http://www.tfl.gov.uk', 'http://www.webmotors.com.br', 'http://www.telangana.gov.in', 'http://www.germanoil.site', 'http://www.popsci.com', 'http://www.digisport.ro', 'http://www.nettavisen.no', 'http://www.shipstation.com', 'http://www.topserialy.to', 'http://www.poloniex.com', 'http://www.arabseed.tv', 'http://www.livecareer.com', 'http://www.yasdl.com', 'http://www.kordramas.com', 'http://www.yamaha.com', 'http://www.waploaded.com', 'http://www.airbnb.ca', 'http://www.eba.gov.tr', 'http://www.rawstory.com', 'http://www.woolworths.com.au', 'http://www.elgiganten.se', 'http://www.nbclosangeles.com', 'http://www.rarbgproxied.org', 'http://www.boursorama.com', 'http://www.guatemala.com', 'http://www.doda.jp', 'http://www.canon.jp', 'http://www.gadgethacks.com', 'http://www.firstonetv.net', 'http://www.torrent-igruha.org', 'http://www.dominos.jp', 'http://www.theblackfriday.com', 'http://www.kitco.com', 'http://www.downloadvideosfrom.com', 'http://www.torrentmovies.co', 'http://www.learncbse.in', 'http://www.securecafe.com', 'http://www.garant.ru', 'http://www.dealabs.com', 'http://www.vseosvita.ua', 'http://www.dashnet.org', 'http://www.prepscholar.com', 'http://www.airarabia.com', 'http://www.sintelevisor.com', 'http://www.dramakoreaindo.com', 'http://www.ingbank.pl', 'http://www.my.gov.au', 'http://www.rtl.fr', 'http://www.psnprofiles.com', 'http://www.1001freefonts.com', 'http://www.hudl.com', 'http://www.metoffice.gov.uk', 'http://www.lachainemeteo.com', 'http://www.startv.com.tr', 'http://www.funnwebs.com', 'http://www.mydailytips.net', 'http://www.siamsport.co.th', 'http://www.onenote.com', 'http://www.webdunia.com', 'http://www.bitfun.co', 'http://www.douploads.com', 'http://www.fotocasa.es', 'http://www.fossil.com', 'http://www.rueducommerce.fr', 'http://www.9xupload.me', 'http://www.fanpop.com', 'http://www.dn.se', 'http://www.xvidzz.com', 'http://www.livecricket.is', 'http://www.guiainfantil.com', 'http://www.semrush.com', 'http://www.ehow.com', 'http://www.vanderbilt.edu', 'http://www.tagesspiegel.de', 'http://www.indodax.com', 'http://www.rarbgget.org', 'http://www.sexvid.xxx', 'http://www.monday.com', 'http://www.michaelkors.com', 'http://www.getcomics.info', 'http://www.showjet.ru', 'http://www.thenounproject.com', 'http://www.prisjakt.nu', 'http://www.shoutmeloud.com', 'http://www.castorama.fr', 'http://www.datafilehost.com', 'http://www.svt.se', 'http://www.digi24.ro', 'http://www.cretalive.gr', 'http://www.zamzar.com', 'http://www.boingboing.net', 'http://www.ani24view.com', 'http://www.dl-protect1.co', 'http://www.shonenjumpplus.com', 'http://www.highlightsfootball.com', 'http://www.mahendras.org', 'http://www.chitai-gorod.ru', 'http://www.pandora.net', 'http://www.lever.co', 'http://www.parstoday.com', 'http://www.mirchi9.com', 'http://www.ioffer.com', 'http://www.postermywall.com', 'http://www.za.gl', 'http://www.swedbank.se', 'http://www.hamusoku.com', 'http://www.poringa.net', 'http://www.3bmeteo.com', 'http://www.putlockertv.to', 'http://www.conrad.de', 'http://www.tuttosport.com', 'http://www.actu.fr', 'http://www.uconn.edu', 'http://www.monash.edu', 'http://www.wacom.com', 'http://www.aritzal.com', 'http://www.shoptime.com.br', 'http://www.rutor.is', 'http://www.lib.ru', 'http://www.bestandfree.com', 'http://www.ustraveldocs.com', 'http://www.telegraaf.nl', 'http://www.vppgamingnetwork.com', 'http://www.pirlotvonline.net', 'http://www.astromeridian.ru', 'http://www.9xbuddy.app', 'http://www.mastermovie-hd.com', 'http://www.pss.pw', 'http://www.kinotravel.co', 'http://www.vibbo.com', 'http://www.aikanqiu.com', 'http://www.theonion.com', 'http://www.weather.gc.ca', 'http://www.uc.edu', 'http://www.4b6994dfa47cee4.com', 'http://www.gdzputina.ru', 'http://www.2cto.com', 'http://www.footmercato.net', 'http://www.codecguide.com', 'http://www.parimatchmirror10.com', 'http://www.freearabsexx.com', 'http://www.chinanetrank.com', 'http://www.afip.gob.ar', 'http://www.obsproject.com', 'http://www.scirp.org', 'http://www.musicnotes.com', 'http://www.sc.com', 'http://www.italki.com', 'http://www.01streamingvf.ws', 'http://www.eatthis.com', 'http://www.seagate.com', 'http://www.dtf.ru', 'http://www.deped.gov.ph', 'http://www.idokep.hu', 'http://www.teepublic.com', 'http://www.rtbf.be', 'http://www.egitimhane.com', 'http://www.xtec.cat', 'http://www.animep.net', 'http://www.motamem.org', 'http://www.mylink.love', 'http://www.streamate.com', 'http://www.plataformaarquitectura.cl', 'http://www.observador.pt', 'http://www.torrental.com', 'http://www.corrieredellosport.it', 'http://www.thepointsguy.com', 'http://www.hardwarezone.com.sg', 'http://www.mac-torrent-download.net', 'http://www.adbsys.icu', 'http://www.engageya.com', 'http://www.gulfnews.com', 'http://www.lloydsbank.com', 'http://www.dandwiki.com', 'http://www.isaimini.co', 'http://www.canadavisa.com', 'http://www.kasvyksta.lt', 'http://www.br.de', 'http://www.airbnb.de', 'http://www.imperiya.by', 'http://www.mihanvideo.com', 'http://www.career.ru', 'http://www.uptostream.com', 'http://www.pcwelt.de', 'http://www.web-hosting.com', 'http://www.phoenixads.co.in', 'http://www.almasryalyoum.com', 'http://www.tjx.com', 'http://www.leroymerlin.es', 'http://www.torrentleech.org', 'http://www.whowatch.tv', 'http://www.pirlotvonline.me', 'http://www.adservme.com', 'http://www.kyodo-d.jp', 'http://www.jobinja.ir', 'http://www.nairabet.com', 'http://www.filmeonline.biz', 'http://www.logmein.com', 'http://www.totalsport.me', 'http://www.pornbus.org', 'http://www.aboutespanol.com', 'http://www.eadaily.com', 'http://www.famitsu.com', 'http://www.arageek.com', 'http://www.publico.pt', 'http://www.torrentikofilm.com', 'http://www.shopdisney.com', 'http://www.kidsa-z.com', 'http://www.gear4music.com', 'http://www.thehackernews.com', 'http://www.kickasstorrents.to', 'http://www.yxdm.tv', 'http://www.slsp.sk', 'http://www.qut.edu.au', 'http://www.tvserial.it', 'http://www.clixsense.com', 'http://www.politpuzzle.ru', 'http://www.frivplus.com', 'http://www.canadapost.ca', 'http://www.olx.ba', 'http://www.luminpdf.com', 'http://www.assistirfilmeshd.me', 'http://www.ulozto.cz', 'http://www.plusone8.com', 'http://www.haqqin.az', 'http://www.allabout.co.jp', 'http://www.las2orillas.co', 'http://www.torrents9.cc', 'http://www.oregonlive.com', 'http://www.saksfifthavenue.com', 'http://www.worldofwarships.eu', 'http://www.fixerinst.com', 'http://www.immonet.de', 'http://www.kiwilimon.com', 'http://www.kariyer.net', 'http://www.gtainside.com', 'http://www.tal.net', 'http://www.mediamarkt.pl', 'http://www.mq.edu.au', 'http://www.ulifestyle.com.hk', 'http://www.timeshighereducation.com', 'http://www.petfinder.com', 'http://www.filmstarts.de', 'http://www.tcsion.com', 'http://www.popularmechanics.com', 'http://www.dizilab.pw', 'http://www.citibank.com', 'http://www.sodapdf.com', 'http://www.findlaw.com', 'http://www.pixieset.com', 'http://www.milfzr.com', 'http://www.vcu.edu', 'http://www.interswitchng.com', 'http://www.lcwaikiki.com', 'http://www.ajoaorexnieym.com', 'http://www.player.pl', 'http://www.cooltext.com', 'http://www.teleman.pl', 'http://www.photokade.com', 'http://www.thestar.com', 'http://www.nasdaq.com', 'http://www.groupon.fr', 'http://www.nch.com.au', 'http://www.ableton.com', 'http://www.gutenberg.org', 'http://www.trueplookpanya.com', 'http://www.bakecaincontrii.com', 'http://www.muni.cz', 'http://www.123doc.org', 'http://www.portalff.is', 'http://www.antaranews.com', 'http://www.freepeople.com', 'http://www.yts.pt', 'http://www.9news.com.au', 'http://www.soujoobafoo.com', 'http://www.royalcaribbean.com', 'http://www.ocn.ne.jp', 'http://www.comic-walker.com', 'http://www.arenabg.com', 'http://www.o2online.de', 'http://www.airbnb.com.au', 'http://www.imlive.com', 'http://www.gala.fr', 'http://www.srt.am', 'http://www.synxis.com', 'http://www.digitec.ch', 'http://www.activision.com', 'http://www.locopelis.com', 'http://www.securecode.com', 'http://www.pdffiller.com', 'http://www.blaxup.org', 'http://www.twilog.org', 'http://www.property24.com', 'http://www.vivo.com.br', 'http://www.opensubtitles.com', 'http://www.psarips.com', 'http://www.appleinsider.com', 'http://www.ut.ac.id', 'http://www.kocpc.com.tw', 'http://www.agah.com', 'http://www.expansion.com', 'http://www.didiaokan.com', 'http://www.calciomercato.com', 'http://www.eanswers.com', 'http://www.fardadl1.net', 'http://www.trend-chaser.com', 'http://www.prensa.com', 'http://www.karnataka.gov.in', 'http://www.myfreeblack.com', 'http://www.ghatreh.com', 'http://www.1lejend.com', 'http://www.51auto.com', 'http://www.bajalogratis.com', 'http://www.seamless.com', 'http://www.bleepingcomputer.com', 'http://www.vfsglobal.com', 'http://www.trome.pe', 'http://www.bettycrocker.com', 'http://www.ovh.com', 'http://www.cinepolis.com', 'http://www.ebanksepah.ir', 'http://www.fel3arda.com', 'http://www.very.co.uk', 'http://www.collabserv.com', 'http://www.nbg.gr', 'http://www.tirerack.com', 'http://www.primosearch.com', 'http://www.eastbay.com', 'http://www.tunisia-sat.com', 'http://www.greatergood.com', 'http://www.televisionlibre.net', 'http://www.tinypng.com', 'http://www.ellegirl.ru', 'http://www.topix.com', 'http://www.stardoll.com', 'http://www.link.tl', 'http://www.vivareal.com.br', 'http://www.bni.co.id', 'http://www.elcinema.com', 'http://www.ebsi.co.kr', 'http://www.tabor.ru', 'http://www.tuoitre.vn', 'http://www.calendly.com', 'http://www.moh.gov.sa', 'http://www.jiumodiary.com', 'http://www.airfrance.fr', 'http://www.bigowu.com', 'http://www.elespectador.com', 'http://www.mytatasky.com', 'http://www.ukraina.ru', 'http://www.yummly.com', 'http://www.filmstreaming1.xyz', 'http://www.estrategiaconcursos.com.br', 'http://www.torhd.com', 'http://www.boards.net', 'http://www.realsimple.com', 'http://www.challonge.com', 'http://www.dlero.net', 'http://www.scoopwhoop.com', 'http://www.urcosme.com', 'http://www.beautifultrendstoday.com', 'http://www.garanti.com.tr', 'http://www.kinotv1.ru', 'http://www.hrdiscussion.com', 'http://www.series9.io', 'http://www.motherjones.com', 'http://www.varlamov.ru', 'http://www.mpgh.net', 'http://www.w24j.com', 'http://www.freepik.es', 'http://www.minecraftservers.org', 'http://www.taqviyat.com', 'http://www.myetherwallet.com', 'http://www.myswitzerland.com', 'http://www.conceptodefinicion.de', 'http://www.alef.ir', 'http://www.xxxmom.pro', 'http://www.savido.net', 'http://www.puzzle-english.com', 'http://www.lyrsense.com', 'http://www.cottonon.com', 'http://www.lnk.fun', 'http://www.defpush.com', 'http://www.unext.jp', 'http://www.reactjs.org', 'http://www.forumcommunity.net', 'http://www.etnet.com.hk', 'http://www.192-168-1-1ip.mobi', 'http://www.lo4d.com', 'http://www.1000.menu', 'http://www.eliittikumppani.fi', 'http://www.admin.ch', 'http://www.allpeliculas.io', 'http://www.dx.com', 'http://www.mylife.com', 'http://www.porngrace.com', 'http://www.4snip.pw', 'http://www.huffingtonpost.es', 'http://www.indianrailways.gov.in', 'http://www.webs.com', 'http://www.uta-net.com', 'http://www.tinkoff.ru', 'http://www.jetairways.com', 'http://www.fussball.de', 'http://www.mouser.com', 'http://www.mnggo.net', 'http://www.domofond.ru', 'http://www.spreadshirt.com', 'http://www.xvideosporno.blog.br', 'http://www.paisdelosjuegos.com.ar', 'http://www.boobpedia.com', 'http://www.totaljobs.com', 'http://www.rotana.video', 'http://www.bestbuylead.com', 'http://www.asos.de', 'http://www.twinfinite.net', 'http://www.fap-nation.com', 'http://www.ofx.to', 'http://www.sendpulse.com', 'http://www.photopea.com', 'http://www.track4ref.com', 'http://www.khatrimazafull.co', 'http://www.peopleperhour.com', 'http://www.onesignal.com', 'http://www.mytube.uz', 'http://www.uic.edu', 'http://www.friday.tw', 'http://www.traidnt.net', 'http://www.buffer.com', 'http://www.zakzak.co.jp', 'http://www.wccftech.com', 'http://www.waveapps.com', 'http://www.365-stream.com', 'http://www.vivastreet.co.uk', 'http://www.nightbot.tv', 'http://www.pond5.com', 'http://www.gordonua.com', 'http://www.svtplay.se', 'http://www.moviescounter.biz', 'http://www.helpshift.com', 'http://www.baur.de', 'http://www.antenam.info', 'http://www.skoob.com.br', 'http://www.coedcherry.com', 'http://www.lpu.in', 'http://www.videospornosos.com', 'http://www.perfil.com', 'http://www.vayava.com', 'http://www.boston.com', 'http://www.kinosezon.tv', 'http://www.pravoslavie.ru', 'http://www.aladin.co.kr', 'http://www.popuble.pro', 'http://www.ralphlauren.com', 'http://www.streaminggratuitvf.com', 'http://www.printful.com', 'http://www.skytorrents.lol', 'http://www.yify.is', 'http://www.strana.ua', 'http://www.uct.ac.za', 'http://www.abw.by', 'http://www.dillards.com', 'http://www.badtameezdilnet.net', 'http://www.ku.dk', 'http://www.indiaporn.me', 'http://www.kojaro.com', 'http://www.mamaclub.com', 'http://www.vod.pl', 'http://www.thepiratefilmeshd.com', 'http://www.collegenet.com', 'http://www.bizpacreview.com', 'http://www.liga.net', 'http://www.askfrank.net', 'http://www.lhy999.com', 'http://www.aftabnews.ir', 'http://www.oasgames.com', 'http://www.kinonix.net', 'http://www.ebah.com.br', 'http://www.menclub.hk', 'http://www.briefly.ru', 'http://www.kickasstorrents.top', 'http://www.nullrefer.com', 'http://www.pxhere.com', 'http://www.kinotan.ru', 'http://www.futbolchile.net', 'http://www.maalaimalar.com', 'http://www.safaribooksonline.com', 'http://www.mtlblog.com', 'http://www.poimel.biz', 'http://www.allmodern.com', 'http://www.bootsnipp.com', 'http://www.mahaonline.gov.in', 'http://www.express.de', 'http://www.300mbfilms.co', 'http://www.audioz.download', 'http://www.aspiration.ai', 'http://www.russkoepornotube.com', 'http://www.trenitalia.com', 'http://www.powtoon.com', 'http://www.vibirai.ru', 'http://www.logitechg.com', 'http://www.ucalgary.ca', 'http://www.dl-xvideos.com', 'http://www.theoutnet.com', 'http://www.delfi.ee', 'http://www.zedo.com', 'http://www.lavozdegalicia.es', 'http://www.etihad.com', 'http://www.scamadviser.com', 'http://www.fotolia.com', 'http://www.workable.com', 'http://www.webtekno.com', 'http://www.skladchina.biz', 'http://www.vshare.eu', 'http://www.prnewswire.com', 'http://www.ufrgs.br', 'http://www.flv2mp3.by', 'http://www.0j7z9aw6.top', 'http://www.quizzstar.com', 'http://www.vodafone.it', 'http://www.libretexts.org', 'http://www.torrentsmd.com', 'http://www.literaturus.ru', 'http://www.pornboil.com', 'http://www.hackernoon.com', 'http://www.techsmith.com', 'http://www.merrjep.al', 'http://www.weatherbug.com', 'http://www.sullca.com', 'http://www.gradescope.com', 'http://www.geneanet.org', 'http://www.oabt004.com', 'http://www.receitasmaster.net', 'http://www.televisionfanatic.com', 'http://www.bt.dk', 'http://www.pho.to', 'http://www.bonanza.com', 'http://www.head-fi.org', 'http://www.desi-serials.org', 'http://www.zoon.ru', 'http://www.putlockerz.io', 'http://www.allkeyshop.com', 'http://www.desidime.com', 'http://www.developpez.net', 'http://www.zerozero.pt', 'http://www.seatgeek.com', 'http://www.gomovies.cool', 'http://www.newsmax.com', 'http://www.concepto-definicion.com', 'http://www.ru-m.org', 'http://www.blacked.com', 'http://www.fremdgehen69.com', 'http://www.macrojuegos.com', 'http://www.kth.se', 'http://www.yimg.com', 'http://www.geocaching.com', 'http://www.gunsamerica.com', 'http://www.mature.nl', 'http://www.lingojam.com', 'http://www.venezuelaaldia.com', 'http://www.pornhub.net', 'http://www.sovets.net', 'http://www.skyozora.com', 'http://www.youtubemp4.to', 'http://www.jobsdb.com', 'http://www.viid.me', 'http://www.qoo10.jp', 'http://www.vk.me', 'http://www.444.hu', 'http://www.2youhd.com', 'http://www.porsche.com', 'http://www.elderscrollsonline.com', 'http://www.douglas.de', 'http://www.cdnquality.com', 'http://www.thanhnien.vn', 'http://www.anitube.site', 'http://www.putlockerhd.is', 'http://www.cera.video', 'http://www.journalistenwatch.com', 'http://www.dr.com.tr', 'http://www.komputerswiat.pl', 'http://www.resheba.com', 'http://www.pretty52.com', 'http://www.bitests.pro', 'http://www.inkapelis.com', 'http://www.shein.co.uk', 'http://www.bakufu.jp', 'http://www.matome-plus.com', 'http://www.lieferando.de', 'http://www.meteredsoftware.com', 'http://www.gowatchseries.co', 'http://www.edusite.ru', 'http://www.x1337x.ws', 'http://www.linuxquestions.org', 'http://www.kino.de', 'http://www.itstillworks.com', 'http://www.cs.money', 'http://www.fullprogramlarindir.com', 'http://www.rtvslo.si', 'http://www.dexonline.ro', 'http://www.modxvm.com', 'http://www.smartprix.com', 'http://www.windguru.cz', 'http://www.paperlesspost.com', 'http://www.affairscloud.com', 'http://www.uservoice.com', 'http://www.enterprise.com', 'http://www.rssing.com', 'http://www.boombastis.com', 'http://www.cryptocompare.com', 'http://www.majorgeeks.com', 'http://www.miradetodo.net', 'http://www.mundosexanuncio.com', 'http://www.videomore.ru', 'http://www.iberia.com', 'http://www.techworm.net', 'http://www.mygully.com', 'http://www.versus.com', 'http://www.lelang.ru', 'http://www.soccer.ru', 'http://www.knockporn.com', 'http://www.thewrap.com', 'http://www.slashdot.org', 'http://www.thedailystar.net', 'http://www.vrt.be', 'http://www.vsetv.com', 'http://www.modcloth.com', 'http://www.gamiss.com', 'http://www.columbia.com', 'http://www.filmon.com', 'http://www.123moviesfun.org', 'http://www.xsrv.jp', 'http://www.epey.com', 'http://www.emisorasunidas.com', 'http://www.sde.co.ke', 'http://www.paisabazaar.com', 'http://www.rasekhoon.net', 'http://www.irrawaddy.com', 'http://www.business-gazeta.ru', 'http://www.wpadx.com', 'http://www.wapmight.net', 'http://www.bestreferat.ru', 'http://www.popupplus.ir', 'http://www.vueling.com', 'http://www.sexmotors.com', 'http://www.univision.com', 'http://www.opfrjkmmvqmm.com', 'http://www.1mg.com', 'http://www.twasul.info', 'http://www.lancers.jp', 'http://www.wisdomjobs.com', 'http://www.bkn.go.id', 'http://www.lamar.edu', 'http://www.massdrop.com', 'http://www.adelaide.edu.au', 'http://www.self.com', 'http://www.natwest.com', 'http://www.bestbooklibrary.com', 'http://www.optifine.net', 'http://www.drmartens.com', 'http://www.ualexa.com', 'http://www.fd1sa1.us', 'http://www.morpakampus.com', 'http://www.pogdesign.co.uk', 'http://www.olx.com.co', 'http://www.kat.rip', 'http://www.autoblog.com', 'http://www.payumoney.com', 'http://www.banyungong.org', 'http://www.x1337x.se', 'http://www.sas.com', 'http://www.parsian-bank.ir', 'http://www.fotomac.com.tr', 'http://www.123movies.solar', 'http://www.yo-movies.com', 'http://www.tripadvisor.com.br', 'http://www.mendeley.com', 'http://www.putlockers.net', 'http://www.ezbuy.sg', 'http://www.vlxx.tv', 'http://www.gitbooks.io', 'http://www.cybersport.ru', 'http://www.medallia.com', 'http://www.send-anywhere.com', 'http://www.fgv.br', 'http://www.open.ac.uk', 'http://www.athome.co.jp', 'http://www.xuexi111.com', 'http://www.keepa.com', 'http://www.gameofglam.com', 'http://www.mee6.xyz', 'http://www.hidratorrent.com', 'http://www.totoria.co', 'http://www.narutoplanet.ru', 'http://www.hollisterco.com', 'http://www.frontier.com', 'http://www.terrikon.com', 'http://www.mailerlite.com', 'http://www.fakt.pl', 'http://www.kmart.com.au', 'http://www.pornicom.com', 'http://www.sbb.ch', 'http://www.demae-can.com', 'http://www.sguru.org', 'http://www.mystart.com', 'http://www.temple.edu', 'http://www.read01.com', 'http://www.carrefour.com.br', 'http://www.deti-online.com', 'http://www.piknu.com', 'http://www.thenationonlineng.net', 'http://www.shape.com', 'http://www.eda.ru', 'http://www.trithucvn.net', 'http://www.altibbi.com', 'http://www.ver-pelis.me', 'http://www.easy-youtube-mp3.com', 'http://www.cryptopia.co.nz', 'http://www.chuansong.me', 'http://www.xe.gr', 'http://www.is.fi', 'http://www.ldlc.com', 'http://www.unian.net', 'http://www.88files.net', 'http://www.fantagazzetta.com', 'http://www.monetizze.com.br', 'http://www.port.hu', 'http://www.mha.gov.in', 'http://www.kinogo.eu', 'http://www.macworld.com', 'http://www.wsu.edu', 'http://www.meownime.com', 'http://www.vgtv.no', 'http://www.xkcd.com', 'http://www.animeid.tv', 'http://www.15min.lt', 'http://www.lyft.com', 'http://www.digikey.com', 'http://www.btbttv.net', 'http://www.harleyquinnwidget.com', 'http://www.with2.net', 'http://www.animedigitalnetwork.fr', 'http://www.nadorcity.com', 'http://www.internshala.com', 'http://www.asiae.co.kr', 'http://www.avid.com', 'http://www.j-cast.com', 'http://www.iltalehti.fi', 'http://www.slant.co', 'http://www.cgtrader.com', 'http://www.xdomain.jp', 'http://www.yahoo-mbga.jp', 'http://www.w3resource.com', 'http://www.threedrive.su', 'http://www.gamestar.de', 'http://www.radiofarda.com', 'http://www.iran-tejarat.com', 'http://www.hlavnespravy.sk', 'http://www.facenama.com', 'http://www.afpbb.com', 'http://www.animeseries.co', 'http://www.xmoviesforyou.com', 'http://www.phys.org', 'http://www.seancody.com', 'http://www.divascancook.com', 'http://www.grabcad.com', 'http://www.gazetevatan.com', 'http://www.animesorion.org', 'http://www.awd.ru', 'http://www.coppel.com', 'http://www.addictinggames.com', 'http://www.surrenderat20.net', 'http://www.talkingpointsmemo.com', 'http://www.rhfgjld.com', 'http://www.9xmovies.org', 'http://www.fark.com', 'http://www.gigafile.nu', 'http://www.pixroute.com', 'http://www.bauhaus.info', 'http://www.goalsarena.org', 'http://www.soescola.com', 'http://www.eztravel.com.tw', 'http://www.timesofisrael.com', 'http://www.finecobank.com', 'http://www.trackerok.org', 'http://www.ugent.be', 'http://www.sportnews.to', 'http://www.education.gouv.fr', 'http://www.ap7am.com', 'http://www.abbp1.com', 'http://www.climatempo.com.br', 'http://www.kogan.com', 'http://www.infojobs.net', 'http://www.dailyinqilab.com', 'http://www.myprotein.com', 'http://www.comingsoon.it', 'http://www.sc.edu', 'http://www.unknownads.com', 'http://www.metrodeal.com', 'http://www.brunch.co.kr', 'http://www.deloton.com', 'http://www.the-star.co.ke', 'http://www.powvideo.net', 'http://www.clipchamp.com', 'http://www.towardsdatascience.com', 'http://www.tovima.gr', 'http://www.picdeer.com', 'http://www.newlook.com', 'http://www.santander.com.br', 'http://www.hiwoenrep.ru', 'http://www.ethikuma.link', 'http://www.yelp.ca', 'http://www.renweb.com', 'http://www.dizipub.co', 'http://www.asp.net', 'http://www.tgju.org', 'http://www.bgr.com', 'http://www.yoreparo.com', 'http://www.artofmanliness.com', 'http://www.cartooncrazy.tv', 'http://www.pearson-intl.com', 'http://www.babytorrent.one', 'http://www.tvball7.com', 'http://www.mac-torrents.com', 'http://www.kaptest.com', 'http://www.i-register.co.in', 'http://www.utiitsl.com', 'http://www.fuskator.com', 'http://www.qxsearch.com', 'http://www.sec.gov', 'http://www.softinggaps.info', 'http://www.load-cloud.icu', 'http://www.dododex.com', 'http://www.paynearby.in', 'http://www.mapion.co.jp', 'http://www.squirt.org', 'http://www.bonappetit.com', 'http://www.freetutorials.eu', 'http://www.24ur.com', 'http://www.benaughty.com', 'http://www.oka.fm', 'http://www.pimpandhost.com', 'http://www.upvid.mobi', 'http://www.dobreprogramy.pl', 'http://www.lexisnexis.com', 'http://www.cyberlink.com', 'http://www.aamc.org', 'http://www.subku.net', 'http://www.ibpsguide.com', 'http://www.dndnha.com', 'http://www.buxinside.com', 'http://www.bcvcrdr.xyz', 'http://www.kinotochka.club', 'http://www.edoc.site', 'http://www.mcafee-br.net', 'http://www.sarahah.com', 'http://www.celebritymoviearchive.com', 'http://www.mathworks.cn', 'http://www.list.am', 'http://www.bkm.com.tr', 'http://www.lanzou.com', 'http://www.myfeed2all.eu', 'http://www.my.com', 'http://www.estrenosby.net', 'http://www.theodysseyonline.com', 'http://www.smartrecruiters.com', 'http://www.hd-streams.org', 'http://www.sports.kz', 'http://www.beenverified.com', 'http://www.fanatical.com', 'http://www.khaosod.co.th', 'http://www.englishclub.com', 'http://www.whatculture.com', 'http://www.kiwi.com', 'http://www.allaboutcircuits.com', 'http://www.obi.de', 'http://www.virgool.io', 'http://www.mk.co.kr', 'http://www.airbnb.es', 'http://www.tournamentsoftware.com', 'http://www.kickass.how', 'http://www.wedisk.co.kr', 'http://www.mtv.com', 'http://www.sexindianvideos.com', 'http://www.movierulzfree.com', 'http://www.owlxxx.com', 'http://www.filmow.com', 'http://www.clarovideo.com', 'http://www.burusoku-vip.com', 'http://www.regmovies.com', 'http://www.webnoviny.sk', 'http://www.kinobar.cc', 'http://www.besthugecocks.com', 'http://www.streamkiste.tv', 'http://www.hobbyconsolas.com', 'http://www.tvb.com', 'http://www.yify-films.net', 'http://www.printfriendly.com', 'http://www.rojadirecta.so', 'http://www.vedomosti.ru', 'http://www.ar15.com', 'http://www.youtubnow.com', 'http://www.fshare.vn', 'http://www.alternate.de', 'http://www.dolartoday.com', 'http://www.lvbet.pl', 'http://www.indeed.fr', 'http://www.sexloving.net', 'http://www.yandex.fr', 'http://www.commonsensemedia.org', 'http://www.greatist.com', 'http://www.escore.gr', 'http://www.aucfan.com', 'http://www.rochester.edu', 'http://www.brainly.com', 'http://www.flyfrontier.com', 'http://www.minimalistbaker.com', 'http://www.rosregistr.ru', 'http://www.revolve.com', 'http://www.orange.es', 'http://www.modelmayhem.com', 'http://www.xl720.com', 'http://www.relistinfo.com', 'http://www.wpolityce.pl', 'http://www.jaiminisbox.com', 'http://www.societe.com', 'http://www.guns.ru', 'http://www.5movies.to', 'http://www.advanceautoparts.com', 'http://www.hyatt.com', 'http://www.aryion.com', 'http://www.eventbrite.co.uk', 'http://www.clixuniverse.com', 'http://www.enhdtv.xyz', 'http://www.meghdadit.com', 'http://www.cloob.com', 'http://www.i-gamer.net', 'http://www.adslgate.com', 'http://www.betin.co.ke', 'http://www.boyner.com.tr', 'http://www.ratopati.com', 'http://www.ub.edu', 'http://www.nicehash.com', 'http://www.bananamall.co.kr', 'http://www.verywellfit.com', 'http://www.wikihow.it', 'http://www.mta.info', 'http://www.jamieoliver.com', 'http://www.hobbylobby.com', 'http://www.tv8.com.tr', 'http://www.netvibes.com', 'http://www.bandicam.com', 'http://www.daz3d.com', 'http://www.lewstream.live', 'http://www.shatelland.com', 'http://www.gate.cc', 'http://www.yougov.com', 'http://www.mediaget.com', 'http://www.neznaka.ru', 'http://www.alfafile.net', 'http://www.inet.se', 'http://www.thepiratebay.vin', 'http://www.postfinance.ch', 'http://www.appuals.com', 'http://www.pi-news.net', 'http://www.gsshop.com', 'http://www.matometanews.com', 'http://www.romwe.com', 'http://www.reserved.com', 'http://www.consequenceofsound.net', 'http://www.lindaikejisblog.com', 'http://www.two-movies.name', 'http://www.18374ir.com', 'http://www.coomeet.com', 'http://www.eslgaming.com', 'http://www.telecinco.es', 'http://www.nic.ir', 'http://www.masbeneficios.life', 'http://www.kinosliva.net', 'http://www.teleprogramma.pro', 'http://www.lastminute.com', 'http://www.elnabaa.net', 'http://www.oi.com.br', 'http://www.clevelandclinic.org', 'http://www.downloadhelper.net', 'http://www.kazagames.com', 'http://www.bookfere.com', 'http://www.dennikn.sk', 'http://www.burporess.pro', 'http://www.undp.org', 'http://www.superlzpre.com', 'http://www.playvids.com', 'http://www.enuri.com', 'http://www.123movies.st', 'http://www.apnetv.co', 'http://www.yehrishtakyakehlata.com', 'http://www.ebook3000.com', 'http://www.wind.it', 'http://www.adbidgo.com', 'http://www.123movies.cafe', 'http://www.reacherinst.com', 'http://www.videosdemadurasx.com', 'http://www.afrikmag.com', 'http://www.kinobos.net', 'http://www.ualberta.ca', 'http://www.hollywoodbets.net', 'http://www.public.gr', 'http://www.pasion.com', 'http://www.my-best.com', 'http://www.qantas.com', 'http://www.glassdoor.co.uk', 'http://www.yenicag.az', 'http://www.indiana.edu', 'http://www.yabaleftonline.ng', 'http://www.torrentcounter.to', 'http://www.vuighe.net', 'http://www.worten.pt', 'http://www.ajc.com', 'http://www.bloombergquint.com', 'http://www.nzz.ch', 'http://www.rrdiscovery.com', 'http://www.logitech.com.cn', 'http://www.juspay.in', 'http://www.shpock.com', 'http://www.oneamour.com', 'http://www.hubspotemail.net', 'http://www.tekstowo.pl', 'http://www.drweb.ru', 'http://www.ssrn.com', 'http://www.trashbox.ru', 'http://www.trainspnrstatus.com', 'http://www.desjardins.com', 'http://www.idealista.it', 'http://www.animeflv.ru', 'http://www.yourarticlelibrary.com', 'http://www.hostinger.com', 'http://www.upmusics.com', 'http://www.soloby.ru', 'http://www.hexafile.net', 'http://www.sparkasse.at', 'http://www.creativecommons.org', 'http://www.pypi.org', 'http://www.aawsat.com', 'http://www.pdx.edu', 'http://www.carleton.ca', 'http://www.statarea.com', 'http://www.toyota.jp', 'http://www.uncommongoods.com', 'http://www.sondakika.com', 'http://www.feide.no', 'http://www.hdtvku.com', 'http://www.peggo.tv', 'http://www.nba4live.fun', 'http://www.maxifoot.fr', 'http://www.sportsv.net', 'http://www.zone-stream.net', 'http://www.gla.ac.uk', 'http://www.auchan.fr', 'http://www.cmjornal.pt', 'http://www.peopletalk.ru', 'http://www.huffingtonpost.kr', 'http://www.fangamer.com', 'http://www.carters.com', 'http://www.icons8.com', 'http://www.scrap.tf', 'http://www.skyscanner.de', 'http://www.pcastuces.com', 'http://www.tugaflix.com', 'http://www.myherbalife.com', 'http://www.internethaber.com', 'http://www.pornlib.com', 'http://www.betensured.com', 'http://www.gamerant.com', 'http://www.domesticatedcompanion.com', 'http://www.atnpx.com', 'http://www.ebgames.ca', 'http://www.expedia.ca', 'http://www.merojax.tv', 'http://www.tomshw.it', 'http://www.flashscore.ro', 'http://www.sdsu.edu', 'http://www.epdf.tips', 'http://www.thetoptens.com', 'http://www.vista.ir', 'http://www.appfolio.com', 'http://www.zalo.me', 'http://www.tvnamu14.me', 'http://www.elkjop.no', 'http://www.linkcaptcha.net', 'http://www.ntnu.no', 'http://www.freee.co.jp', 'http://www.soccerline.kr', 'http://www.nd.edu', 'http://www.navient.com', 'http://www.busyteacher.org', 'http://www.destinytracker.com', 'http://www.jobstreet.co.id', 'http://www.hastidownloadz.info', 'http://www.learnthelyrics.com', 'http://www.zalora.co.id', 'http://www.junkmail.co.za', 'http://www.webuntis.com', 'http://www.qanon.pub', 'http://www.blogosfera.uol.com.br', 'http://www.ibctamil.com', 'http://www.wired.jp', 'http://www.banamex.com', 'http://www.mastersportal.com', 'http://www.officeworks.com.au', 'http://www.scholarship-positions.com', 'http://www.rarbgunblock.com', 'http://www.18qt.com', 'http://www.otorrents.com', 'http://www.eddirasa.com', 'http://www.isaidub.net', 'http://www.brainly.ph', 'http://www.iranestekhdam.ir', 'http://www.bmew2.com', 'http://www.arsenal.com', 'http://www.englisch-hilfen.de', 'http://www.patoghu.com', 'http://www.adultwork.com', 'http://www.phoneinfox.com', 'http://www.bet365.it', 'http://www.85nian.net', 'http://www.dnevnik.hr', 'http://www.vikatan.com', 'http://www.meneame.net', 'http://www.hankyung.com', 'http://www.falabella.com', 'http://www.upstore.net', 'http://www.hdnora.ru', 'http://www.facebook.github.io', 'http://www.fender.com', 'http://www.livehotty.com', 'http://www.wikibuy.com', 'http://www.rit.edu', 'http://www.searchnu.com', 'http://www.looperman.com', 'http://www.ns21.club', 'http://www.lotterysambad.com', 'http://www.autobild.de', 'http://www.filmakinesi.net', 'http://www.deccanchronicle.com', 'http://www.unisa.ac.za', 'http://www.ac-versailles.fr', 'http://www.eatingwell.com', 'http://www.watchsomuch.info', 'http://www.incestflix.com', 'http://www.trivago.com', 'http://www.nadarenews.com', 'http://www.depositfiles.com', 'http://www.polar.com', 'http://www.ideacellular.com', 'http://www.toggl.com', 'http://www.vippers.jp', 'http://www.istgah.com', 'http://www.imbc.com', 'http://www.downloadly.ir', 'http://www.paisdelosjuegos.com.mx', 'http://www.uafilm.tv', 'http://www.hotfiesta.tv', 'http://www.nrttv.com', 'http://www.ilmkidunya.com', 'http://www.przelewy24.pl', 'http://www.drakorindo.cc', 'http://www.bsnl.co.in', 'http://www.rumble.com', 'http://www.shafaqna.com', 'http://www.xmtrading.com', 'http://www.bikroy.com', 'http://www.zevent.fr', 'http://www.telemagazyn.pl', 'http://www.seslisozluk.net', 'http://www.vodxc.com', 'http://www.desiupload.in', 'http://www.destinyitemmanager.com', 'http://www.thzbt.co', 'http://www.j-lyric.net', 'http://www.tufts.edu', 'http://www.kupdf.net', 'http://www.prokerala.com', 'http://www.finviz.com', 'http://www.keepvid.com', 'http://www.makeshop.jp', 'http://www.putlockers.plus', 'http://www.aesoponline.com', 'http://www.ingdirect.es', 'http://www.debtv.ru', 'http://www.dist-app.com', 'http://www.deichmann.com', 'http://www.tudelft.nl', 'http://www.nordea.fi', 'http://www.varsitytutors.com', 'http://www.apkhome.net', 'http://www.quickheal.com', 'http://www.kkbox.com', 'http://www.geni.com', 'http://www.moz.com', 'http://www.natalie.mu', 'http://www.vova.com', 'http://www.r-project.org', 'http://www.xxxvideos247.com', 'http://www.tinypic.com', 'http://www.pximg.net', 'http://www.escapefromtarkov.com', 'http://www.lookfantastic.com', 'http://www.amc.com', 'http://www.hdkinoteatr.com', 'http://www.thegospelcoalition.org', 'http://www.getemoji.com', 'http://www.netsuite.com', 'http://www.inaporn.com', 'http://www.elgiganten.dk', 'http://www.dn.pt', 'http://www.typing.com', 'http://www.subscene.xyz', 'http://www.adslzone.net', 'http://www.cam4.es', 'http://www.chochox.com', 'http://www.imgfile.net', 'http://www.datezone.com', 'http://www.weziwezi.com', 'http://www.persianblog.ir', 'http://www.expresstabloid.ba', 'http://www.tvsod.com', 'http://www.rackspace.com', 'http://www.shoes.com', 'http://www.banglarbhumi.gov.in', 'http://www.linkzb.net', 'http://www.parsine.com', 'http://www.magoosh.com', 'http://www.bitrix24.com', 'http://www.rug.nl', 'http://www.calibre-ebook.com', 'http://www.novafile.com', 'http://www.islamway.net', 'http://www.desiasm.club', 'http://www.enpareja.com', 'http://www.your-surveys.com', 'http://www.bibliaonline.com.br', 'http://www.chollometro.com', 'http://www.voeazul.com.br', 'http://www.saudiairlines.com', 'http://www.jugem.jp', 'http://www.rulertube.com', 'http://www.netonnet.se', 'http://www.universityofcalifornia.edu', 'http://www.scarlet-clicks.info', 'http://www.supremenewyork.com', 'http://www.guardaserie.watch', 'http://www.dibamovie1.com', 'http://www.pstatic.net', 'http://www.kikinote.net', 'http://www.adexchangecloud.com', 'http://www.embedy.cc', 'http://www.foxitsoftware.com', 'http://www.rbc.ua', 'http://www.mynewspepper.com', 'http://www.favorit.com.ua', 'http://www.wootalk.today', 'http://www.annunci69.it', 'http://www.isport.ua', 'http://www.mmorpg.com', 'http://www.myprotein.jp', 'http://www.gimy.tv', 'http://www.clasohlson.com', 'http://www.metafilter.com', 'http://www.songhouse.me', 'http://www.fstoppers.com', 'http://www.zumiez.com', 'http://www.allboxing.ru', 'http://www.haaretz.com', 'http://www.globalbankingandfinance.com', 'http://www.pornid.xxx', 'http://www.livesicilia.it', 'http://www.gametracker.com', 'http://www.citizensbankonline.com', 'http://www.nudography.com', 'http://www.mmajunkie.com', 'http://www.academy.com', 'http://www.lovoo.com', 'http://www.newspicks.com', 'http://www.mazolporn.com', 'http://www.expedia.co.jp', 'http://www.coindesk.com', 'http://www.dbrand.com', 'http://www.klix.ba', 'http://www.video.az', 'http://www.nabble.com', 'http://www.truecar.com', 'http://www.fileis.com', 'http://www.clavier-arab.org', 'http://www.heroku.com', 'http://www.moovitapp.com', 'http://www.goal.in.th', 'http://www.docsity.com', 'http://www.care.com', 'http://www.apyecom.com', 'http://www.nation.co.ke', 'http://www.islatively.com', 'http://www.stockcharts.com', 'http://www.masteringphysics.com', 'http://www.videoquizstar.com', 'http://www.clk.sh', 'http://www.repaik.com', 'http://www.mb.com.ph', 'http://www.pornogratisdiario.com', 'http://www.m24.ru', 'http://www.convert-video-online.com', 'http://www.tchibo.de', 'http://www.parsnews.com', 'http://www.quoka.de', 'http://www.powerbi.com', 'http://www.trendymen.ru', 'http://www.barclaycard.co.uk', 'http://www.vysledkyvolieb.sk', 'http://www.112.ua', 'http://www.mytvsuper.com', 'http://www.allwomens.ru', 'http://www.protect-your-privacy-now.com', 'http://www.iarex.ru', 'http://www.alitools.io', 'http://www.xtream-codes.com', 'http://www.aitiden.net', 'http://www.minnstate.edu', 'http://www.walkintubs.zone', 'http://www.bitdefender.com', 'http://www.seeking.com', 'http://www.aleteia.org', 'http://www.clipart-library.com', 'http://www.slashfilm.com', 'http://www.unionmangas.site', 'http://www.dir.bg', 'http://www.sublimetext.com', 'http://www.swisscom.ch', 'http://www.jkanime.net', 'http://www.bankmandiri.co.id', 'http://www.fineartamerica.com', 'http://www.ktrmr.com', 'http://www.sinoptik.com.ru', 'http://www.wifistudy.com', 'http://www.songtexte.com', 'http://www.jogos.uol.com.br', 'http://www.5terka.com', 'http://www.te.eg', 'http://www.mensjournal.com', 'http://www.besthdmovies.me', 'http://www.classes.ru', 'http://www.merkur.de', 'http://www.bankofindia.com', 'http://www.novaskin.me', 'http://www.gry-online.pl', 'http://www.jomashop.com', 'http://www.clickssp.pro', 'http://www.soshistagram.com', 'http://www.uniquecaptcha.com', 'http://www.homeadvisor.com', 'http://www.serasaconsumidor.com.br', 'http://www.postmates.com', 'http://www.itresan.com', 'http://www.alltrails.com', 'http://www.fow.kr', 'http://www.mnogohd.com', 'http://www.raidbots.com', 'http://www.visa.com', 'http://www.nashol.com', 'http://www.jotform.com', 'http://www.etudier.com', 'http://www.scikit-learn.org', 'http://www.bloody-disgusting.com', 'http://www.rowfirst.eu', 'http://www.fanpelis.com', 'http://www.bonusbitcoin.co', 'http://www.movavi.ru', 'http://www.flixbus.com', 'http://www.vtrahevip.tv', 'http://www.all.biz', 'http://www.blackmagicdesign.com', 'http://www.blikk.hu', 'http://www.fendi.com', 'http://www.download.com.vn', 'http://www.comss.ru', 'http://www.angelfire.com', 'http://www.scielo.org.mx', 'http://www.blackdesertonline.com', 'http://www.cpanel.net', 'http://www.ahcdn.com', 'http://www.facenews.ua', 'http://www.giracoin.com', 'http://www.tdameritrade.com', 'http://www.hitta.se', 'http://www.rarlab.com', 'http://www.leeds.ac.uk', 'http://www.taishinbank.com.tw', 'http://www.unknowncheats.me', 'http://www.mercadolibre.com.uy', 'http://www.vccs.edu', 'http://www.ixigo.com', 'http://www.vrfuckdolls.com', 'http://www.buzzadnetwork.com', 'http://www.atube.me', 'http://www.pdftoimage.com', 'http://www.fonbet.ru', 'http://www.eva.ru', 'http://www.openrice.com', 'http://www.brown.edu', 'http://www.promiedos.com.ar', 'http://www.cutw.pro', 'http://www.gilabola.com', 'http://www.filmpertutti.uno', 'http://www.bitrix24.ru', 'http://www.csun.edu', 'http://www.aldi-sued.de', 'http://www.iomovies.to', 'http://www.coco02.net', 'http://www.ufmg.br', 'http://www.tokyotosho.info', 'http://www.nationalreview.com', 'http://www.depaul.edu', 'http://www.megastudy.net', 'http://www.atv.com.tr', 'http://www.jpopsuki.eu', 'http://www.pricena.com', 'http://www.oglaf.com', 'http://www.avon.com', 'http://www.saudia.com', 'http://www.neakriti.gr', 'http://www.dailyxing.com', 'http://www.tarjetarojatvonline.com', 'http://www.webstaurantstore.com', 'http://www.polri.go.id', 'http://www.lapalingo.com', 'http://www.fakings.com', 'http://www.wikium.ru', 'http://www.koreaboo.com', 'http://www.oriflame.ru', 'http://www.vndb.org', 'http://www.banco.bradesco', 'http://www.luisaviaroma.com', 'http://www.sunsu521.com', 'http://www.bestdeals.today', 'http://www.bibliofond.ru', 'http://www.funpay.ru', 'http://www.hrazens.com', 'http://www.filmesonline.zone', 'http://www.sport365.live', 'http://www.m4ufree.tv', 'http://www.smarturl.it', 'http://www.clegc-gckey.gc.ca', 'http://www.3ddd.ru', 'http://www.actionnewsnow.com', 'http://www.ohyeah1080.com', 'http://www.drexel.edu', 'http://www.depop.com', 'http://www.fanat.tv', 'http://www.filmesonlinehd1.net', 'http://www.sarvgyan.com', 'http://www.cppreference.com', 'http://www.life.hu', 'http://www.drive2.com', 'http://www.bang.com', 'http://www.kali.org', 'http://www.wyzant.com', 'http://www.analyticsvidhya.com', 'http://www.askmen.com', 'http://www.beststopapne.com', 'http://www.vidxxx.info', 'http://www.0daydown.com', 'http://www.random.org', 'http://www.pdalife.ru', 'http://www.embibe.com', 'http://www.cultofmac.com', 'http://www.teenpornvideo.xxx', 'http://www.textures.com', 'http://www.anonymz.com', 'http://www.gimp.org', 'http://www.filmzor.net', 'http://www.buenastareas.com', 'http://www.porntv.com', 'http://www.zalora.sg', 'http://www.babnet.net', 'http://www.boerse.to', 'http://www.mindbodygreen.com', 'http://www.milli.az', 'http://www.espn.com.br', 'http://www.freelogoservices.com', 'http://www.flydubai.com', 'http://www.180chan.co', 'http://www.denik.cz', 'http://www.ustaliy.ru', 'http://www.tax.service.gov.uk', 'http://www.rincondelvago.com', 'http://www.empowr.com', 'http://www.chowhound.com', 'http://www.nist.gov', 'http://www.cinetux.to', 'http://www.townhall.com', 'http://www.lpsg.com', 'http://www.hemnet.se', 'http://www.instantcheckmate.com', 'http://www.isubtitles.info', 'http://www.lci.fr', 'http://www.launchpad.net', 'http://www.salefiles.com', 'http://www.cardkingdom.com', 'http://www.idsly.org', 'http://www.savoirsnumeriques5962.fr', 'http://www.haber61.net', 'http://www.lapresse.ca', 'http://www.gradesaver.com', 'http://www.ooreka.fr', 'http://www.surfearner.com', 'http://www.gazetadopovo.com.br', 'http://www.jio.in', 'http://www.xunta.gal', 'http://www.lurkmore.to', 'http://www.vroomvroomz.com', 'http://www.clubedohardware.com.br', 'http://www.btcliving.com', 'http://www.worldpay.com', 'http://www.iloveimg.com', 'http://www.ru-minecraft.ru', 'http://www.downloadplayer.xyz', 'http://www.worthpoint.com', 'http://www.awok.com', 'http://www.ilgazzettino.it', 'http://www.seodollars.com', 'http://www.nejm.org', 'http://www.tvonline.id', 'http://www.qmap.pub', 'http://www.wallstreetoasis.com', 'http://www.day.az', 'http://www.prisjakt.no', 'http://www.spigotmc.org', 'http://www.tudogostoso.com.br', 'http://www.ddl-warez.to', 'http://www.audiofanzine.com', 'http://www.snhu.edu', 'http://www.hs.fi', 'http://www.game-debate.com', 'http://www.runnersworld.com', 'http://www.diziyo4.net', 'http://www.smbc-comics.com', 'http://www.filejo.com', 'http://www.banggood.in', 'http://www.hollywoodlife.com', 'http://www.applyweb.com', 'http://www.yourtemplatefinder.com', 'http://www.inverse.com', 'http://www.seatguru.com', 'http://www.bytefence.com', 'http://www.exawarosu.net', 'http://www.searchprivacy.co', 'http://www.adrunnr.com', 'http://www.piratebays.be', 'http://www.watchfree.ac', 'http://www.sb24.com', 'http://www.abcya.com', 'http://www.xvideos-br.com', 'http://www.zedge.net', 'http://www.customink.com', 'http://www.speedtest.cn', 'http://www.mamahd.org', 'http://www.btcclicks.com', 'http://www.thinkresult.in', 'http://www.sportsmole.co.uk', 'http://www.openstreetmap.org', 'http://www.heroesofthestorm.com', 'http://www.timeedit.net', 'http://www.vitacost.com', 'http://www.alphapolis.co.jp', 'http://www.enallaktikidrasi.com', 'http://www.kanopy.com', 'http://www.bancosantander.es', 'http://www.tvline.com', 'http://www.girls888tk.tk', 'http://www.mytransitguide.com', 'http://www.krakow.pl', 'http://www.justia.com', 'http://www.forvo.com', 'http://www.daily.mk', 'http://www.mo5577.net', 'http://www.rgho.st', 'http://www.hentaifapland.com', 'http://www.rxlist.com', 'http://www.libaclub.com', 'http://www.postquare.com', 'http://www.codeplex.com', 'http://www.concordia.ca', 'http://www.mytheresa.com', 'http://www.lybrate.com', 'http://www.rulate.ru', 'http://www.comodo.com', 'http://www.voenhronika.ru', 'http://www.wikidata.org', 'http://www.free3d.com', 'http://www.jwplayer.com', 'http://www.ad.nl', 'http://www.dubiwang.com', 'http://www.flyscoot.com', 'http://www.4porn.com', 'http://www.mlive.com', 'http://www.getuploader.com', 'http://www.blueletterbible.org', 'http://www.1porno.online', 'http://www.avatan.ru', 'http://www.eprice.com.hk', 'http://www.trivago.in', 'http://www.nu.edu.bd', 'http://www.glassdoor.ca', 'http://www.technopat.net', 'http://www.mql5.com', 'http://www.freeofread.com', 'http://www.familyhandyman.com', 'http://www.usamodelkina.ru', 'http://www.pasted.co', 'http://www.motor1.com', 'http://www.volvocars.com', 'http://www.muzonov.net', 'http://www.steelseries.com', 'http://www.libertatea.ro', 'http://www.atf42.com', 'http://www.ubar-pro4.ru', 'http://www.dorkly.com', 'http://www.pusan.ac.kr', 'http://www.rice.edu', 'http://www.wufoo.com', 'http://www.neuvoo.com', 'http://www.annonceintime.com', 'http://www.uvm.edu', 'http://www.costcotravel.com', 'http://www.cb2.com', 'http://www.sjsu.edu', 'http://www.bobs-tube.com', 'http://www.hotschedules.com', 'http://www.sss.xxx', 'http://www.imx.to', 'http://www.arcai.com', 'http://www.thenightseries.net', 'http://www.bnf.fr', 'http://www.esea.net', 'http://www.torrentboza.com', 'http://www.rmz.cr', 'http://www.studylib.net', 'http://www.hipertextual.com', 'http://www.elementor.com', 'http://www.trabalhosfeitos.com', 'http://www.gaijinpot.com', 'http://www.adpgtr.com', 'http://www.shon.xyz', 'http://www.foxtel.com.au', 'http://www.ebravo.pk', 'http://www.hryssc.in', 'http://www.rover.com', 'http://www.sunhome.ru', 'http://www.babyou.com', 'http://www.geoguessr.com', 'http://www.xcite.com', 'http://www.myunidays.com', 'http://www.arminius.io', 'http://www.mp3indirdur.mobi', 'http://www.massimodutti.com', 'http://www.mediamarkt.com.tr', 'http://www.imagesnake.com', 'http://www.turktelekom.com.tr', 'http://www.cbexams.com', 'http://www.crohasit.com', 'http://www.redditgifts.com', 'http://www.uukanshu.com', 'http://www.portail.free.fr', 'http://www.streema.com', 'http://www.thesims.com', 'http://www.proxyrarbg.org', 'http://www.maturealbum.com', 'http://www.nflstream.io', 'http://www.vtorrente.org', 'http://www.aflam4you.tv', 'http://www.loveplanet.ru', 'http://www.osdn.net', 'http://www.styl.pl', 'http://www.best-torrent.net', 'http://www.azvideo.net', 'http://www.blogcms.jp', 'http://www.ing.nl', 'http://www.desapegacaxias.com', 'http://www.bytrd.info', 'http://www.frontiersin.org', 'http://www.harfetaze.com', 'http://www.1xstavka.ru', 'http://www.freepornq.com', 'http://www.tmdb.org', 'http://www.hypoot.com', 'http://www.minkch.com', 'http://www.pancakeapps.com', 'http://www.topasnew.com', 'http://www.mtgsalvation.com', 'http://www.alakhbarmedia.net', 'http://www.cpuid.com', 'http://www.nla.gov.au', 'http://www.alibabacloud.com', 'http://www.findprice.com.tw', 'http://www.arubanetworks.com', 'http://www.luc.edu', 'http://www.login.gov', 'http://www.singaporeair.com', 'http://www.pinkbike.com', 'http://www.evga.com', 'http://www.bertina.us', 'http://www.musavat.com', 'http://www.deref-mail.com', 'http://www.chainreactioncycles.com', 'http://www.destinygamewiki.com', 'http://www.portaleducativo.net', 'http://www.fontmeme.com', 'http://www.allover30.com', 'http://www.glgoo.top', 'http://www.seekdiscomfort.com', 'http://www.tonymacx86.com', 'http://www.diziizlesen1.com', 'http://www.ee.co.uk', 'http://www.twenga.de', 'http://www.imagefruit.com', 'http://www.powershow.com', 'http://www.himasoku.com', 'http://www.pikdo.net', 'http://www.gettyimages.co.uk', 'http://www.biggamez.com', 'http://www.mytelevisionhq.com', 'http://www.abc13.com', 'http://www.convertunits.com', 'http://www.agregadornews.com', 'http://www.militarytimes.com', 'http://www.milligazete.com.tr', 'http://www.explosm.net', 'http://www.vatera.hu', 'http://www.yad2.co.il', 'http://www.premiumbeat.com', 'http://www.hqtube.xxx', 'http://www.benesse.ne.jp', 'http://www.eligasht.com', 'http://www.depositfiles.org', 'http://www.shoudian.org', 'http://www.nash-dom2.su', 'http://www.yonsei.ac.kr', 'http://www.asianxv.com', 'http://www.seekyourfortune.info', 'http://www.lipstickalley.com', 'http://www.kech24.com', 'http://www.ruclip.com', 'http://www.cheathappens.com', 'http://www.kathimerini.gr', 'http://www.msdmanuals.com', 'http://www.sandisk.com', 'http://www.jamb.org.ng', 'http://www.globalreachtech.com', 'http://www.mediaonly.ru', 'http://www.travelchinaguide.com', 'http://www.id-olymptrade.com', 'http://www.hearthstonetopdecks.com', 'http://www.dmitory.com', 'http://www.hermo.my', 'http://www.nautiljon.com', 'http://www.kinonews.ru', 'http://www.unibet.be', 'http://www.subscribe.ru', 'http://www.lulus.com', 'http://www.jumia.co.ke', 'http://www.streamelements.com', 'http://www.imobie.com', 'http://www.startlap.hu', 'http://www.filmehd.net', 'http://www.realgm.com', 'http://www.8591.com.tw', 'http://www.swedbank.lt', 'http://www.docplayer.es', 'http://www.rockauto.com', 'http://www.homeaway.com', 'http://www.104.ua', 'http://www.dtu.dk', 'http://www.siterdm.com', 'http://www.runtastic.com', 'http://www.giantitp.com', 'http://www.iyfnzgb.com', 'http://www.ororo.tv', 'http://www.patefon.net', 'http://www.olx.ph', 'http://www.whosampled.com', 'http://www.mlsmatrix.com', 'http://www.fnbr.co', 'http://www.eros.com', 'http://www.caracoltv.com', 'http://www.pearltrees.com', 'http://www.fileinfo.com', 'http://www.labnol.org', 'http://www.kitty-kats.net', 'http://www.freehdx.com', 'http://www.prodigygame.com', 'http://www.liberty.edu', 'http://www.hkepc.com', 'http://www.milenio.com', 'http://www.arrhow.xyz', 'http://www.zr.ru', 'http://www.techlandia.com', 'http://www.rtbravo.com', 'http://www.b3l3l.com', 'http://www.wotspeak.ru', 'http://www.wegotthiscovered.com', 'http://www.s-dm.com', 'http://www.whois.com', 'http://www.pleer.ru', 'http://www.flightclub.com', 'http://www.serienjunkies.org', 'http://www.4book.org', 'http://www.porncomixonline.net', 'http://www.ragalahari.com', 'http://www.pdfescape.com', 'http://www.phun.org', 'http://www.biografiasyvidas.com', 'http://www.bayunblocked.eu', 'http://www.elephanttube.com', 'http://www.monbureaunumerique.fr', 'http://www.elcuz.com', 'http://www.picksandparlays.net', 'http://www.cgv.co.kr', 'http://www.hpe.com', 'http://www.movie4k.to', 'http://www.worldsources.com', 'http://www.thesiliconreview.com', 'http://www.freevideo.cz', 'http://www.cgd.pt', 'http://www.vfxdownload.com', 'http://www.nifty.org', 'http://www.euro-football.ru', 'http://www.fl.ru', 'http://www.ck12.org', 'http://www.freeforums.net', 'http://www.cdromance.com', 'http://www.xvideo-jp.com', 'http://www.diablofans.com', 'http://www.hinative.com', 'http://www.rarbgaccessed.org', 'http://www.soccersuck.com', 'http://www.news-postseven.com', 'http://www.upoint.id', 'http://www.vintage-erotica-forum.com', 'http://www.lavoz.com.ar', 'http://www.plejada.pl', 'http://www.canna.to', 'http://www.hwcdn.net', 'http://www.kritika24.ru', 'http://www.universa.uol.com.br', 'http://www.ugm.ac.id', 'http://www.seneporno.com', 'http://www.studentdoctor.net', 'http://www.allmovies.uz', 'http://www.larazon.es', 'http://www.dauid-iep.com', 'http://www.cbse.nic.in', 'http://www.motivation8success.info', 'http://www.sayidaty.net', 'http://www.aol.de', 'http://www.inkbunny.net', 'http://www.qooqootv.pro', 'http://www.155chan.gr', 'http://www.turboserial.net', 'http://www.splinternews.com', 'http://www.solarmovie.id', 'http://www.arabysexy.com', 'http://www.zkillboard.com', 'http://www.pro-gaming-world.com', 'http://www.kaspi.kz', 'http://www.neko-miku.com', 'http://www.surfconext.nl', 'http://www.u-car.com.tw', 'http://www.egexa.com', 'http://www.allbooksy.com', 'http://www.newmobilelife.com', 'http://www.europaplus.ru', 'http://www.listenonrepeat.com', 'http://www.telegraf.rs', 'http://www.soft32.com', 'http://www.teamspeak.com', 'http://www.directlinkkpush.com', 'http://www.nwolb.com', 'http://www.fiu.edu', 'http://www.prettylittlething.com', 'http://www.carrefour.fr', 'http://www.bitcoin.com', 'http://www.y3600.co', 'http://www.twister.porn', 'http://www.wimp.com', 'http://www.onepiece-ex.com.br', 'http://www.tecmint.com', 'http://www.torrentz2.is', 'http://www.hket.com', 'http://www.berliner-sparkasse.de', 'http://www.onamae.com', 'http://www.nts.org.pk', 'http://www.dicasdeviagens.me', 'http://www.segundamano.mx', 'http://www.thermofisher.com', 'http://www.blankmediagames.com', 'http://www.royallib.com', 'http://www.ecartelera.com', 'http://www.catch.com.au', 'http://www.tahrirnews.com', 'http://www.cardgames.io', 'http://www.cheers.com.tw', 'http://www.pakwheels.com', 'http://www.fuckmoral.com', 'http://www.kingsoft.jp', 'http://www.americatv.com.pe', 'http://www.virtualbox.org', 'http://www.game01.ru', 'http://www.behindthename.com', 'http://www.freeroms.com', 'http://www.teacup.com', 'http://www.cafef.vn', 'http://www.ni.com', 'http://www.betway.com.gh', 'http://www.agenziaentrate.gov.it', 'http://www.tcd.ie', 'http://www.modiseh.com', 'http://www.danimados.com', 'http://www.miradalta.me', 'http://www.ykt.ru', 'http://www.au.dk', 'http://www.clickpost.jp', 'http://www.travel.co.jp', 'http://www.linkprice.com', 'http://www.javfree.me', 'http://www.restream.io', 'http://www.astrosage.com', 'http://www.rawmangazip.com', 'http://www.etoro.com', 'http://www.jagobd.com', 'http://www.adrenaline.uol.com.br', 'http://www.getmedia.zone', 'http://www.hoichoi.tv', 'http://www.simsdom.com', 'http://www.ittefaq.com.bd', 'http://www.huffingtonpost.it', 'http://www.onlineclock.net', 'http://www.panopto.com', 'http://www.torrentool.com', 'http://www.exceljet.net', 'http://www.cdbaby.com', 'http://www.playstream.co', 'http://www.babelio.com', 'http://www.songmeanings.com', 'http://www.doktorsitesi.com', 'http://www.hktvmall.com', 'http://www.narutoget.ru', 'http://www.paultan.org', 'http://www.upbom.live', 'http://www.rogerebert.com', 'http://www.bloodmallet.com', 'http://www.moradu.com', 'http://www.softgozar.com', 'http://www.muji.net', 'http://www.twrp.me', 'http://www.shopgoodwill.com', 'http://www.simplyhired.com', 'http://www.gamesfull.org', 'http://www.kompass.com', 'http://www.getdroidtips.com', 'http://www.adreactor.com', 'http://www.fasttech.com', 'http://www.vsemayki.ru', 'http://www.doityourself.com', 'http://www.ac-lille.fr', 'http://www.king.com', 'http://www.mall.cz', 'http://www.rg-mechanics.org', 'http://www.suny.edu', 'http://www.justclick.ru', 'http://www.playcast.ru', 'http://www.filecloud.io', 'http://www.uesp.net', 'http://www.indimusic.tv', 'http://www.overclockers.ru', 'http://www.youpit.site', 'http://www.tv2.dk', 'http://www.photoshop-master.ru', 'http://www.moxing.mba', 'http://www.helloidol.com', 'http://www.assistirseriados.net', 'http://www.ultimateclassicrock.com', 'http://www.sho.com', 'http://www.sportchek.ca', 'http://www.nrega.nic.in', 'http://www.bri.co.id', 'http://www.srstop.online', 'http://www.genbeta.com', 'http://www.paidverts.com', 'http://www.gkcdbc.com', 'http://www.airliners.net', 'http://www.pdfcandy.com', 'http://www.womany.net', 'http://www.topito.com', 'http://www.skyscanner.es', 'http://www.purepeople.com', 'http://www.kaspersky.ru', 'http://www.alpha.gr', 'http://www.foxsports.com.au', 'http://www.wanikani.com', 'http://www.indavideo.hu', 'http://www.plannedparenthood.org', 'http://www.wapka.mobi', 'http://www.hayneedle.com', 'http://www.koton.com', 'http://www.uu.se', 'http://www.avm.de', 'http://www.piratebayproxy.info', 'http://www.governmentjobs.com', 'http://www.cj.com', 'http://www.qu.edu.qa', 'http://www.reed.co.uk', 'http://www.ifood.com.br', 'http://www.freakarcade.com', 'http://www.zoominfo.com', 'http://www.payletter.com', 'http://www.indiegala.com', 'http://www.snapfish.com', 'http://www.erodate.pl', 'http://www.avadl.me', 'http://www.jonathandedecker.com', 'http://www.groupon.co.uk', 'http://www.tunefind.com', 'http://www.adintrend.com', 'http://www.filmapik.xyz', 'http://www.cooch.tv', 'http://www.olimpiada.ru', 'http://www.warthunder.ru', 'http://www.wikizeroo.net', 'http://www.sportybet.com', 'http://www.portalfinalfantasy.com', 'http://www.ert.gr', 'http://www.support.wix.com', 'http://www.myperfectresume.com', 'http://www.tportal.hr', 'http://www.hostloc.com', 'http://www.berlin.de', 'http://www.mcafee-br.org', 'http://www.kinguin.net', 'http://www.worlanned.pro', 'http://www.merksjbgrpyb.download', 'http://www.peing.net', 'http://www.dovidka.biz.ua', 'http://www.w.org', 'http://www.insomnia.gr', 'http://www.davinsurance.com', 'http://www.mutually.com', 'http://www.cvut.cz', 'http://www.unt.edu', 'http://www.weeb.tv', 'http://www.androidmtk.com', 'http://www.jamejamonline.ir', 'http://www.syfy.com', 'http://www.venturebeat.com', 'http://www.synonym.com', 'http://www.sportschatplace.com', 'http://www.astroportal.com', 'http://www.corel.com', 'http://www.moluch.ru', 'http://www.hiamag.com', 'http://www.panzerrush.com', 'http://www.clubensayos.com', 'http://www.oecd.org', 'http://www.youtube-video-download.com', 'http://www.lectio.dk', 'http://www.eddiebauer.com', 'http://www.khodrobank.com', 'http://www.mastercuriosidadesbr.com', 'http://www.rogers.com', 'http://www.teachingenglish.org.uk', 'http://www.diep.io', 'http://www.anaconda.com', 'http://www.pravda.ru', 'http://www.se.pl', 'http://www.col3negoriginal.lk', 'http://www.betano.com', 'http://www.uanl.mx', 'http://www.resumegenius.com', 'http://www.driverscape.com', 'http://www.letudiant.fr', 'http://www.hotlink.cc', 'http://www.learningapps.org', 'http://www.tocana.jp', 'http://www.graftaub.com', 'http://www.flashscores.co.uk', 'http://www.nvidia.ru', 'http://www.hdqwalls.com', 'http://www.corporatefinanceinstitute.com', 'http://www.cricketaddictor.com', 'http://www.fantia.jp', 'http://www.chunxiao.tv', 'http://www.anastasiadate.com', 'http://www.hentaifox.com', 'http://www.gardeningknowhow.com', 'http://www.ikshow.net', 'http://www.vodafone.es', 'http://www.koreanair.com', 'http://www.bitbounce.com', 'http://www.gamerescape.com', 'http://www.franceculture.fr', 'http://www.xvideos0.blog.br', 'http://www.rule34hentai.net', 'http://www.picarto.tv', 'http://www.easyen.ru', 'http://www.godlikeproductions.com', 'http://www.eurosportplayer.com', 'http://www.appllio.com', 'http://www.tuttocampo.it', 'http://www.momjunction.com', 'http://www.downsub.com', 'http://www.archives.gov', 'http://www.eldorar.com', 'http://www.turkpress.co', 'http://www.umuc.edu', 'http://www.pornhat.com', 'http://www.outlookabsorb.com', 'http://www.dollskill.com', 'http://www.geogebra.org', 'http://www.linkskeep.xyz', 'http://www.buffalo.jp', 'http://www.solidworks.com', 'http://www.companieshouse.gov.uk', 'http://www.moosejaw.com', 'http://www.amdm.ru', 'http://www.talktalk.co.uk', 'http://www.ashleyfurniture.com', 'http://www.comproliverton.pro', 'http://www.hentaila.com', 'http://www.farzand.net', 'http://www.dm.de', 'http://www.lunapic.com', 'http://www.laborlaw.com.cn', 'http://www.kokobum.com', 'http://www.epizy.com', 'http://www.niche.com', 'http://www.redirectowl.com', 'http://www.t24.com.tr', 'http://www.linode.com', 'http://www.cuti.in', 'http://www.hertz.com', 'http://www.powerpyx.com', 'http://www.travelfriend.online', 'http://www.sony.net', 'http://www.zattoo.com', 'http://www.edu-74.ru', 'http://www.formstack.com', 'http://www.yasour.org', 'http://www.surfline.com', 'http://www.ntvspor.net', 'http://www.zennioptical.com', 'http://www.monoprice.com', 'http://www.bookfi.net', 'http://www.astro.com', 'http://www.adverts.ie', 'http://www.watan.com', 'http://www.caltech.edu', 'http://www.kleientertainment.com', 'http://www.wordstream.com', 'http://www.vrporn.com', 'http://www.muzlostyle.ru', 'http://www.newspapers.com', 'http://www.smartasset.com', 'http://www.totalwar.com', 'http://www.programme-television.org', 'http://www.gem-flash.com', 'http://www.ssrmovies.cc', 'http://www.compass.education', 'http://www.wayfair.co.uk', 'http://www.onclickbright.com', 'http://www.downrose.com', 'http://www.overclockers.co.uk', 'http://www.bombuj.eu', 'http://www.toonkor.one', 'http://www.fileflares.com', 'http://www.mp4.ir', 'http://www.53.com', 'http://www.younow.com', 'http://www.teknosa.com', 'http://www.comfy.ua', 'http://www.vu.edu.pk', 'http://www.virtualdj.com', 'http://www.pa.gov', 'http://www.fontanka.ru', 'http://www.brusheezy.com', 'http://www.minecraftforge.net', 'http://www.meijer.com', 'http://www.hemailaccessonline.com', 'http://www.hellosehat.com', 'http://www.hsoub.com', 'http://www.imperial.ac.uk', 'http://www.epublibre.org', 'http://www.roaring.earth', 'http://www.becu.org', 'http://www.jimmyjohns.com', 'http://www.akhbaralaan.net', 'http://www.libero.pe', 'http://www.odyssys.net', 'http://www.zapimoveis.com.br', 'http://www.bet.pt', 'http://www.gymshark.com', 'http://www.mirknig.su', 'http://www.liveresult.ru', 'http://www.zynga.com', 'http://www.qrz.com', 'http://www.uottawa.ca', 'http://www.saucenao.com', 'http://www.shaw.ca', 'http://www.italia-film.pro', 'http://www.ps7894.com', 'http://www.foroactivo.com', 'http://www.bk33.mobi', 'http://www.bisnis.com', 'http://www.free-cloud-storage.com', 'http://www.firstcry.com', 'http://www.gentside.com', 'http://www.jillsclickcorner.com', 'http://www.adsl.free.fr', 'http://www.poradnikzdrowie.pl', 'http://www.hotnews.ro', 'http://www.pnas.org', 'http://www.rotogrinders.com', 'http://www.4j.com', 'http://www.iconarchive.com', 'http://www.ebay.be', 'http://www.ideapod.com', 'http://www.vidlox.tv', 'http://www.umnaja.ru', 'http://www.imgcarry.com', 'http://www.nationalrail.co.uk', 'http://www.wego.com', 'http://www.ankaraport.net', 'http://www.canarabank.com', 'http://www.tvnet.lv', 'http://www.gomlab.com', 'http://www.t3n.de', 'http://www.lolzteam.net', 'http://www.qpdownload.com', 'http://www.tourister.ru', 'http://www.assrt.net', 'http://www.datumglobal.com', 'http://www.awwwards.com', 'http://www.dl-zip.com', 'http://www.ants.gouv.fr', 'http://www.adventurefeeds.com', 'http://www.mdr.de', 'http://www.mckinsey.com', 'http://www.s7.ru', 'http://www.tetrisfriends.com', 'http://www.fashionguide.com.tw', 'http://www.manicomio-share.com', 'http://www.hartgeld.com', 'http://www.neowin.net', 'http://www.dailynayadiganta.com', 'http://www.imageban.ru', 'http://www.yesbank.in', 'http://www.predictiondisplay.com', 'http://www.turkcell.com.tr', 'http://www.classlifestyle.com', 'http://www.bwin.it', 'http://www.megapornfreehd.com', 'http://www.gingersoftware.com', 'http://www.radio.com', 'http://www.giveitlove.com', 'http://www.yammer.com', 'http://www.morningstar.com', 'http://www.openlibrary.org', 'http://www.payu.com', 'http://www.sumomo-ch.com', 'http://www.sopawfect.com', 'http://www.wmich.edu', 'http://www.downloadelements.com', 'http://www.iotvet.com', 'http://www.gmatclub.com', 'http://www.femina.mk', 'http://www.realityblurb.com', 'http://www.gameinformer.com', 'http://www.japan-baseball.jp', 'http://www.kari.com', 'http://www.sparkfun.com', 'http://www.hqxmovies.com', 'http://www.dealplatter.com', 'http://www.shine.com', 'http://www.gestiopolis.com', 'http://www.pia.jp', 'http://www.tube.ac', 'http://www.mirnovostey.info', 'http://www.getsurl.com', 'http://www.gov.on.ca', 'http://www.izooto.com', 'http://www.iranbanou.com', 'http://www.voyeur-house.tv', 'http://www.cultbeauty.co.uk', 'http://www.arabapps.me', 'http://www.hi.gt', 'http://www.82cook.com', 'http://www.three.co.uk', 'http://www.worksmobile.com', 'http://www.bolha.com', 'http://www.dailyfunnyworld.com', 'http://www.mikrotik.com', 'http://www.qafqazinfo.az', 'http://www.unibytes.com', 'http://www.takprosto.cc', 'http://www.cadenaser.com', 'http://www.netfullfilmizle.com', 'http://www.sheknows.com', 'http://www.expediapartnercentral.com', 'http://www.blackhatworld.com', 'http://www.blitz.bg', 'http://www.fun-mooc.fr', 'http://www.getyourguide.com', 'http://www.manytorrents.pro', 'http://www.miniinthebox.com', 'http://www.webmusic.live', 'http://www.4kstreams.net', 'http://www.interneturok.ru', 'http://www.putlockers.fm', 'http://www.thesprucecrafts.com', 'http://www.cgsociety.org', 'http://www.colaboraread.com.br', 'http://www.playerauctions.com', 'http://www.difi.no', 'http://www.colonelcassad.livejournal.com', 'http://www.electronics-tutorials.ws', 'http://www.torrentking.eu', 'http://www.apornoflv.tv', 'http://www.roozno.com', 'http://www.puer.org.ua', 'http://www.priceprice.com', 'http://www.s-oman.net', 'http://www.owlcation.com', 'http://www.putlocker.dance', 'http://www.fap18.net', 'http://www.perfectmoney.is', 'http://www.stmods.ru', 'http://www.blueporns.com', 'http://www.streamcherry.com', 'http://www.swedbank.lv', 'http://www.alfajertv.com', 'http://www.avaeduc.com.br', 'http://www.mapsgalaxy.com', 'http://www.tripadvisor.jp', 'http://www.3dsky.org', 'http://www.literaguru.ru', 'http://www.torrentpong.com', 'http://www.reg.ru', 'http://www.comdotgame.com', 'http://www.adidas.ru', 'http://www.91app.com', 'http://www.diariocorreo.pe', 'http://www.bibliaon.com', 'http://www.snu.ac.kr', 'http://www.indane.co.in', 'http://www.9111.ru', 'http://www.beyondmenu.com', 'http://www.filmitorrentom.org', 'http://www.zain.com', 'http://www.simplenote.com', 'http://www.coderanch.com', 'http://www.digifinex.com', 'http://www.asianetnews.com', 'http://www.tubitv.com', 'http://www.znanio.ru'] +INDIA_SITES = ['http://100india.com/', 'http://105matures.com', 'http://18teensexposed.tumblr.com', 'http://2gayboys.com', 'http://3rat.com', 'http://4hen.com', 'http://6mature9.com', 'http://action36.com', 'http://adultfriendfinder.com', 'http://adultwork.com', 'http://aflatune.com/', 'http://ahmilf.com', 'http://alansanal.com', 'http://allindiansexstories.com', 'http://amapics.net', 'http://amateurmaturewives.com', 'http://amateur-sexys.tumblr.com', 'http://andheriescorts.org.in', 'http://ankita-ahuja.in', 'http://apniisp.com/', 'http://awesomeellalove.tumblr.com', 'http://axnxxx.org', 'http://babesclub.net', 'http://bamapachyderm.com/', 'http://bananabunny.com', 'http://beautiful-nude-teens-exposed.tumblr.com', 'http://beautyandthebeard1.tumblr.com', 'http://beeg.com', 'http://bestmaturesthumb.com', 'http://bestmaturewomen.com', 'http://bigdickswillingchicks.tumblr.com', 'http://bigtitsfree.net', 'http://bigtitsporn.me', 'http://bollymaza.com/', 'http://bollywoodmp4.com/', 'http://boobymilf.com', 'http://broslingerie.com', 'http://cckerala.com/', 'http://charmingmumbai.com', 'http://chaturbate.com', 'http://chimatamusic.com/', 'http://chirkutonorkut.com/', 'http://cleoteener.com', 'http://coffetube.com', 'http://coolgoose.com/', 'http://cuckinohio.tumblr.com', 'http://cutepornvideos.com', 'http://depositfiles.com/', 'http://des-filles-sexy.com', 'http://desijammers.com/', 'http://desindian.sextgem.com', 'http://desisexclips.com', 'http://desishock.net/', 'http://dhool.com/', 'http://dildosatisfaction.tumblr.com', 'http://dishant.com/', 'http://dslady.com', 'http://eatyouout.tumblr.com', 'http://en.cam4.co', 'http://en.cam4.com.br', 'http://eroticperfection.com', 'http://es.bongacams.com', 'http://es.bravotube.net', 'http://es.chaturbate.com', 'http://escortsbaba.com/mumbai-escorts.html', 'http://escortservicemumbai.co.in', 'http://es.porn.com', 'http://excitingmatures.com', 'http://fantasticwomans.com', 'http://filmicafe.com/', 'http://freesex.com', 'http://freex.mobi', 'http://fr-nostradamus.com', 'http://fr.perfectgirls.net', 'http://fuckdc.com', 'http://fuckmaturewhore.com', 'http://fuck-milf.com', 'http://fuckmycheatingslutwife.tumblr.com', 'http://gaypornium.com', 'http://girlmature.com', 'http://girthyencounters.tumblr.com', 'http://godao.com', 'http://gogrumogru.com/', 'http://golden-moms.com', 'http://gracefulnudes.com', 'http://gramateurs.com', 'http://h33t.to', 'http://hairy.com', 'http://happy-ending.biz', 'http://hindiold.com', 'http://hindi-sex.net', 'http://hollywoodjizz.com', 'http://hot-dates.info', 'http://hotelmatures.com', 'http://hotfreemilfs.com', 'http://hothomemadepix.tumblr.com', 'http://hot-naked-milfs.com', 'http://hqlinks.net', 'http://hungrymatures.com', 'http://icematures.com', 'http://iloveindiansex.com', 'http://ilovematurewomen.tumblr.com', 'http://imagessearchyahoo.com/', 'http://imaturewomen.com', 'http://immoralmatures.com', 'http://ishitamalhotra.com', 'http://jaanu.co.in/mumbai-escorts-service-call-girls.html', 'http://jasmineescorts.com', 'http://jellyroll.biz', 'http://jizzporntube.com', 'http://jlobster.com', 'http://juicygranny.com', 'http://juicylips.biz', 'http://juicy-matures.com', 'http://kalyn.in', 'http://kavyajain.in', 'http://kellydivine.co', 'http://khushikapoor.in', 'http://kiiran.in/', 'http://kirtu.com', 'http://krazywap.com/', 'http://lenawethole.com', 'http://lipkiss.site', 'http://lisaannlovers11.tumblr.com', 'http://live.sugarbbw.com', 'http://lookatvintage.com', 'http://lovemaking.co.in', 'http://lovepaki.com/', 'http://mackers-world.com/', 'http://mastmag.com/', 'http://mature30plus.com', 'http://mature4.net', 'http://matureamour.com', 'http://mature-beach.com', 'http://matureclithunter.com', 'http://matureclits.net', 'http://maturedolls.net', 'http://maturegirl.us', 'http://matureinlove.net', 'http://maturenags.com', 'http://maturenakedsluts.com', 'http://matureoracle.com', 'http://matureplace.com', 'http://maturepornhub.com', 'http://maturesexy.us', 'http://matureshine.com', 'http://maturesort.com', 'http://matures-photos.com', 'http://maturewant.com', 'http://mature-women-tube.net', 'http://mature-women-tube.org', 'http://megamovie.us', 'http://mega-teen.com', 'http://milfatwork.net', 'http://milfera.com', 'http://milffreepictures.com', 'http://milf-fucking.net', 'http://milfgals.net', 'http://milfionaire.com', 'http://milfous.com', 'http://milfpornet.com', 'http://mobidreamz.com/', 'http://mom50.com', 'http://momsinporn.net', 'http://momspics.net', 'http://monikabas.co.in', 'http://motherless.com', 'http://motherstits.com', 'http://mulligansmilfs.com', 'http://mumbaiescorts.co.in', 'http://mumbai-escorts.info', 'http://mumbai-escorts.leathercurrency.com', 'http://muskurahat.com/', 'http://myexmilf.com', 'http://myfreemoms.com', 'http://mypornbookmarks.com', 'http://newsfilter.org', 'http://nudeboobshotpics.com', 'http://nudedares.tumblr.com', 'http://oigh.info', 'http://oldernastybitches.com', 'http://oldsexybabes.net', 'http://oldwomanface.com', 'http://onlyoneescorts.com', 'http://owsmut.com', 'http://pahubad.com', 'http://pinkvisualtgp.com', 'http://playboy.com', 'http://porn720.com', 'http://pornnakedgirls.com', 'http://porno-wife.com', 'http://pornxxxtubes.com', 'http://porny.com', 'http://priyasen.in', 'http://punjabcentral.com/', 'http://queenofmature.com', 'http://radioreloaded.com/', 'http://realitypassplus.com', 'http://riomature.com', 'http://riomoms.com', 'http://riyaverma.in', 'http://rkmania.com/', 'http://role-play.biz', 'http://rubinakapoor.biz', 'http://samwep.com/', 'http://savitabhabhi.mobi', 'http://sendspace.com/', 'http://sevanthi.com/', 'http://sex3.com', 'http://sexocean.com', 'http://sexpics.xxx', 'http://sextubelinks.com', 'http://sexxxxi.com', 'http://sexymaturepics.com', 'http://sexymaturepussies.com', 'http://shemale.asia', 'http://shimi.in', 'http://shitbrix.com', 'http://shneha.in', 'http://silkymoms.com', 'http://smutty.com', 'http://songdad.com/', 'http://songsnonstop.com/', 'http://spicyfm.com/', 'http://spicymumbai.in', 'http://stiflersmilfs.com', 'http://stiflersmoms.com', 'http://stretchedpussy.tumblr.com', 'http://sunomusic.com/', 'http://tamilkey.info/', 'http://teen18ass.com', 'http://teengayporntube.com', 'http://tgpmaturewoman.com', 'http://thefreecamsecret.com', 'http://thehotpics.com', 'http://themomsfucking.net', 'http://thenisai.com/', 'http://thepiratescove.us/', 'http://theporndude.com', 'http://tiny-cams.com', 'http://uniquesexymoms.com', 'http://unshavenpussies.net', 'http://upskirttop.net', 'http://vashiescorts.co.in/', 'http://videos.petardas.com', 'http://videos-porno.x18xxx.com', 'http://viewmature.com', 'http://vintagehairy.net', 'http://vipoldies.net', 'http://vivthomas.com', 'http://wapindia.net/', 'http://webcam.com', 'http://webpnudes.com', 'http://wetmaturewhores.com', 'http://wetmaturewomen.com', 'http://whoresmilfsdegraded.tumblr.com', 'http://wifenaked.net', 'http://womenmaturepics.com', 'http://woodstockreborn.tumblr.com', 'http://www.30yomilf.com', 'http://www.3movs.com', 'http://www.40somethingmag.com', 'http://www.69rueporno.com', 'http://www.7feel.net', 'http://www.89.com', 'http://www.8nsex.com', 'http://www.ad2earn.com/', 'http://www.adultphonechatlines.co.uk', 'http://www.agedcunts.net', 'http://www.agedmamas.com', 'http://www.ah-me.com', 'http://www.alexmatures.com', 'http://www.all-free-nude-old-granny-mature-women-xxx-porn-pics.com', 'http://www.allindiansex.com', 'http://www.alloldpics.com', 'http://www.alohatube.com', 'http://www.analsexstars.com', 'http://www.antarvasna.com', 'http://www.arabesexy.com', 'http://www.asiatique-femme.com', 'http://www.askyourmommy.com', 'http://www.ass4all.com', 'http://www.ass-butt.com', 'http://www.azgals.com', 'http://www.aztecaporno.com', 'http://www.babes.com', 'http://www.babosas.com', 'http://www.bbwpornpics.com', 'http://www.beeg.co', 'http://www.befuck.com', 'http://www.bestmatureclips.com', 'http://www.bestmilftube.com', 'http://www.bigbuttmature.com', 'http://www.bigtinz.com', 'http://www.bigtitsmilf.com', 'http://www.bigtitsnaked.com', 'http://www.bollywood-sex.net', 'http://www.bravomamas.com', 'http://www.brazzers.com', 'http://www.brazzersnetwork.com', 'http://www.bubblebuttpics.com', 'http://www.bullporn.com', 'http://www.bushypussies.net', 'http://www.cerdas.com', 'http://www.cheatwife.com', 'http://www.chubbygalls.com', 'http://www.chubbygirlpics.com', 'http://www.cindymovies.com', 'http://www.cliphunter.com', 'http://www.clit7.com', 'http://www.cloaktube.com', 'http://www.cocomilfs.com', 'http://www.culx.org', 'http://www.darering.com', 'http://www.desikahani.net', 'http://www.desikamasutra.com', 'http://www.dianapost.com', 'http://www.digitalplayground.com', 'http://www.dreammovies.com', 'http://www.drtuber.com', 'http://www.dustyporn.com', 'http://www.ebonyfantasies.com', 'http://www.eeltube.com', 'http://www.elderly-women.com', 'http://www.empflix.com', 'http://www.epicporntube.com', 'http://www.eromatures.net', 'http://www.eroticbeauties.net', 'http://www.eroticteens.pw', 'http://www.eternaldesire.com', 'http://www.fatsexygirls.net', 'http://www.filthymamas.com', 'http://www.filthyoldies.com', 'http://www.find-best-lingerie.com', 'http://www.find-best-videos.com', 'http://www.flirt4free.com', 'http://www.flirthookup.com', 'http://www.freekiloclips.com', 'http://www.freematurepornpics.com', 'http://www.freemilfpornpics.com', 'http://www.freemilfsite.com', 'http://www.freeones.ch', 'http://www.freeones.com', 'http://www.free-porn-pics.net', 'http://www.freshporn.info', 'http://www.fuckcuck.com', 'http://www.fucking8.com', 'http://www.fuckmaturewhore.com', 'http://www.funjadu.com', 'http://www.fuq.com', 'http://www.galsarchive.com', 'http://www.gay43.com', 'http://www.gaytube.com', 'http://www.gggay.com', 'http://www.gonzo.com', 'http://www.gonzoxxxmovies.com', 'http://www.gracefulmilf.com', 'http://www.gracefulmom.com', 'http://www.gracefulnudes.com', 'http://www.grandmammapics.com', 'http://www.grannyhairy.net', 'http://www.grannypornpics.net', 'http://www.guide-asie.com', 'http://www.hairy.com', 'http://www.hairymilfpics.com', 'http://www.hardsextube.com', 'http://www.hindisex.com', 'http://www.hinduhumanrights.org/hindufocus.html', 'http://www.hinduunity.net/', 'http://www.hinduunity.org/', 'http://www.horny-olders.com', 'http://www.hotmomspics.com', 'http://www.hotmomsporn.com', 'http://www.hotnakedoldies.com', 'http://www.hotnudematures.com', 'http://www.hotsexyteensphotos.com', 'http://www.hqmaturemovs.com', 'http://www.hqoldies.com', 'http://www.idealmilf.com', 'http://www.idealwifes.com', 'http://www.iknowthatgirl.com', 'http://www.imagefap.com', 'http://www.imamali8.com/', 'http://www.immodestmoms.com', 'http://www.imomsex.com', 'http://www.indiankahani.com', 'http://www.indianpornovid.com', 'http://www.indiansgoanal.org', 'http://www.indienne-sexy.com', 'http://www.iscindia.org', 'http://www.ixxx.com', 'http://www.ixxx-tube.com', 'http://www.jeux-flash-sexy.com', 'http://www.jizzhut.com', 'http://www.jizzle.com', 'http://www.juliepost.com', 'http://www.katestube.com', 'http://www.keezmovies.com', 'http://www.kilopics.com', 'http://www.kink.com', 'http://www.kissmaturesgo.com', 'http://www.labatidora.net', 'http://www.ladymom.com', 'http://www.lewd-girls.com', 'http://www.lewdmistress.com', 'http://www.liberteenage.com', 'http://www.locasporfollar.com', 'http://www.lushstories.com', 'http://www.lustfuloldies.com', 'http://www.maghrebinnes.xl.cx', 'http://www.mature30-45.com', 'http://www.matureal.com', 'http://www.matureandgranny.com', 'http://www.matureandyoung.com', 'http://www.matureasspics.com', 'http://www.maturecherry.net', 'http://www.maturecool.com', 'http://www.maturehotsex.com', 'http://www.matureintros.com', 'http://www.matureladiespics.com', 'http://www.matureland.net', 'http://www.mature-library.com', 'http://www.maturemompics.com', 'http://www.maturemomsex.com', 'http://www.mature-orgasm.com', 'http://www.maturepornhere.com', 'http://www.maturepornpics.com', 'http://www.maturepornqueens.net', 'http://www.maturetube.com', 'http://www.maturosexy.com', 'http://www.mc-nudes.com', 'http://www.megavideoporno.org', 'http://www.milfbank.com', 'http://www.milfionaire.com', 'http://www.milfjam.com', 'http://www.milfkiss.com', 'http://www.milfmomspics.com', 'http://www.milfmovs.com', 'http://www.milfparanoia.com', 'http://www.milfpicshere.com', 'http://www.milfsarea.com', 'http://www.milfsbeach.com', 'http://www.momhandjob.com', 'http://www.mommyxxxmovies.com', 'http://www.momsclan.com', 'http://www.momsecstasy.com', 'http://www.momshere.com', 'http://www.momstaboo.com', 'http://www.mrskin.com', 'http://www.mulligansmilfs.com', 'http://www.muyzorras.com', 'http://www.myfreecams.com', 'http://www.myhdshop.com', 'http://www.mynakedteens.com', 'http://www.myonlyhd.com', 'http://www.nakedboobs.net', 'http://www.naked-moms.com', 'http://www.naughty.com', 'http://www.nndh.com/', 'http://www.nudeartstars.com', 'http://www.nudematurepussy.com', 'http://www.nudematurespics.com', 'http://www.nudematurewomenphotos.com', 'http://www.nudemomandboy.com', 'http://www.nudemomphotos.com', 'http://www.nude-oldies.com', 'http://www.nudevista.com', 'http://www.nuvid.com', 'http://www.older-beauty.com', 'http://www.oldercherry.com', 'http://www.olderkiss.com', 'http://www.oldmomstgp.com', 'http://www.oldpoon.com', 'http://www.oldsweet.com', 'http://www.old-vulva.com', 'http://www.onlygirlvideos.com', 'http://www.onlyporngif.com', 'http://www.pamelapost.com', 'http://www.pandamovies.com', 'http://www.perfectgirls.net', 'http://www.perucaseras.com', 'http://www.petardas.com', 'http://www.picsboob.com', 'http://www.pinkteenpics.com', 'http://www.pinkworld.com', 'http://www.place21.com', 'http://www.playboy.com', 'http://www.playvid.com', 'http://www.pof.com', 'http://www.poringa.net', 'http://www.porn.com', 'http://www.pornhub.com', 'http://www.pornmaturepics.com', 'http://www.pornmaturewomen.com', 'http://www.porno.com', 'http://www.pornodoido.com', 'http://www.pornorama.com', 'http://www.pornorc.net', 'http://www.pornoriver.net', 'http://www.pornstarhangout.com', 'http://www.pornsticky.com', 'http://www.porntubevidz.com', 'http://www.premiercastingporno.com', 'http://www.primecurves.com', 'http://www.purebbwtube.com', 'http://www.pussy-mature.com', 'http://www.rahulyadav.com/', 'http://www.randyhags.com', 'http://www.redtube.com', 'http://www.riomature.com', 'http://www.riomilf.com', 'http://www.riomoms.com', 'http://www.riotits.net', 'http://www.rk.com', 'http://www.roundandbrown.com', 'http://www.sambaporno.com', 'http://www.secinsurance.com', 'http://www.secretarypics.com', 'http://www.sexocean.com', 'http://www.sexxxdoll.com', 'http://www.sexybuttpics.com', 'http://www.sexyhotmilf.com', 'http://www.sexyhotmilfs.com', 'http://www.sexy-links.net', 'http://www.sexymilfpussy.com', 'http://www.sexynakedamateurgirls.com', 'http://www.sexyteensphotos.com', 'http://www.sharedxpics.com', 'http://www.slutload.com', 'http://www.spankwire.com', 'http://www.specialgays.com', 'http://www.stripping-moms.com', 'http://www.sweetmaturepics.com', 'http://www.teencamvids.org', 'http://www.teenpornxxx.net', 'http://www.teenpussy.pw', 'http://www.teensnow.com', 'http://www.teensnowxvideos.com', 'http://www.tgpmaturewoman.com', 'http://www.thefreecamsecret.com', 'http://www.thehotpics.com', 'http://www.theunity.org/', 'http://www.thexmilf.com', 'http://www.tinysolo.com', 'http://www.tnaflix.com', 'http://www.toonztube.com', 'http://www.toroporno.com', 'http://www.truthordarepics.com', 'http://www.tube8.com', 'http://www.tubegogo.com', 'http://www.tubepornstars.com', 'http://www.tubestack.com', 'http://www.tubexclips.com', 'http://www.uniquesexymoms.com', 'http://www.universeold.com', 'http://www.unshavengirls.net', 'http://www.upskirt.com', 'http://www.videosdemadurasx.com', 'http://www.videospornonacional.com', 'http://www.visarev.com/', 'http://www.wetmaturepics.com', 'http://www.wifesbank.com', 'http://www.wifezilla.com', 'http://www.womanolder.com', 'http://www.womeninyears.com', 'http://www.womenmaturepics.com', 'http://www.worldxxxphotos.com', 'http://www.x-art.com', 'http://www.xebonygirls.com', 'http://www.x-ho.com', 'http://www.xmilfpics.com', 'http://www.xnxx.com', 'http://www.xnxxgifs.com', 'http://www.xtube.com', 'http://www.xvideos.com', 'http://www.xvideosnacional.com', 'http://www.xxx.com', 'http://www.xxxmomclips.com', 'http://www.xxxvideosex.org', 'http://www.yasminramos.com', 'http://www.yehfun.com', 'http://www.youjizz.com', 'http://www.youngpornvideos.com', 'http://www.youngxxxpics.com', 'http://www.youporn.com', 'http://www.yourlustgirlfriends.com', 'http://xbabe.com', 'http://xhot.sextgem.com', 'http://xnxx-free.net', 'http://xnxx.vc', 'http://xxxbunker.com', 'http://xxx.com', 'http://xxxsummer.net', 'http://youjizz.ws', 'http://youngmint.com', 'http://zinkwap.com/'] diff --git a/plugins/http/plugin.py b/plugins/http/plugin.py new file mode 100644 index 0000000..d0a8add --- /dev/null +++ b/plugins/http/plugin.py @@ -0,0 +1,263 @@ +""" +HTTP Plugin driver + +Overrides the default evaluator plugin handling so we can negotiate a clear port +and track jailed sites to avoid residual censorship. +""" + +import argparse +import calendar +import copy +import logging +import os +import random +import socket +import subprocess as sp +import sys +import tempfile +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import engine +import external_sites +import actions.utils + +from plugins.plugin import Plugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(os.path.dirname(BASEPATH)) +TEST_SITES = copy.deepcopy(external_sites.EXTERNAL_SITES) +JAIL_TRACKER = {} +for site in TEST_SITES: + JAIL_TRACKER[site] = 0 +random.shuffle(TEST_SITES) + +GOOD_WORD = "testing" +BAD_WORD = "ultrasurf" +JAIL_TIME = 95 + + +class HTTPPluginRunner(Plugin): + """ + Defines the HTTP plugin runner. + """ + name = "http" + def __init__(self, args): + """ + Marks this plugin as enabled + """ + self.enabled = True + + def negotiate_clear_port(self, args, evaluator, environment, logger): + """ + Since residual censorship might be affecting our IP/port combo + that was randomly chosen, this method is to find a port on which + no residual censorship is present. This is done simply by picking a + port, running the server, running curl to confirm it's accessible, + and then returning that port. + """ + while True: + # Pick a random port if port negotiation is enabled + if args["disable_port_negotiation"] and not args.get("censor"): + port = 80 + else: + port = random.randint(1025, 65000) + port_bound = True + while port_bound: + try: + with socket.socket() as sock: + sock.bind(('', port)) + # if no error thrown, port was unbound and thus we can use it + port_bound = False + except OSError: + # port was bound already, generate new port and try again + port = random.randint(1025, 65000) + + # Store that port in the server args + evaluator.server_args.update({"port": port}) + # Disable the engine so the strategy under test does not interfere with the + # residual censorship check + evaluator.server_args.update({"no_engine": True}) + + # Start the server on our chosen port + try: + http_server = evaluator.start_server(evaluator.server_args, environment, logger) + except: + logger.exception("Failed to start server; choosing a new port.") + continue + + if args["disable_port_negotiation"] or args.get("censor"): + return http_server, port + + # Test for residual censorship + dest = "%s:%d" % (evaluator.get_ip(), port) + logger.debug("Checking for residual censorship at %s" % dest) + command = "curl -s %s -m 5 -v" % dest + stdout, stderr = evaluator.remote_exec_cmd(environment["remote"], command, logger, timeout=7) + for line in stderr: + if "Connection reset by peer" in line: + logger.info("Residual censorship detected on %s." % dest) + evaluator.stop_server(environment, http_server) + time.sleep(1) + continue + elif "timed out" in line: + logger.debug("Connection timed out on %s" % dest) + evaluator.stop_server(environment, http_server) + raise actions.utils.SkipStrategyException("Strategy broke TCP connection", -400) + break + return http_server, port + + def start(self, args, evaluator, environment, ind, logger): + """ + Runs the plugins + """ + if args["use_external_sites"]: + args.update({"external_server" : True}) + + forwarder = {} + if evaluator.act_as_middlebox: + forwarder["sender_ip"] = args.get("sender_ip") + forwarder["forward_ip"] = args.get("forward_ip") + forwarder["routing_ip"] = args.get("routing_ip") + + port = args.get("port", 80) + tmp_dir = None + # If we're given a server to start, start it now, but if we're a middlebox, don't run a server + if evaluator.server_cls and not args.get("external_server") and not evaluator.act_as_middlebox: + server, port = self.negotiate_clear_port(args, evaluator, environment, logger) + + # Update the port with the given or negotiated port + evaluator.client_args.update({"port": port}) + site_to_test = evaluator.client_args.get("server", "") + + output_path = os.path.join(PROJECT_ROOT, evaluator.client_args.get("output_directory")) + + with engine.Engine(port, args.get("strategy", ""), server_side=args["server_side"], environment_id=environment["id"], output_directory=output_path, log_level=args.get("log", "debug"), enabled=args["server_side"], forwarder=forwarder) as eng: + with TestServer(site_to_test, evaluator, environment, logger) as site_to_test: + evaluator.client_args.update({"server" : site_to_test}) + fitness = evaluator.run_client(evaluator.client_args, environment, logger) + + evaluator.read_fitness(ind) + + # If the engine ran on the server side, ask that it punish fitness + if args["server_side"]: + ind.fitness = actions.utils.punish_fitness(fitness, logger, eng) + actions.utils.write_fitness(ind.fitness, output_path, environment["id"]) + + if evaluator.server_cls and not evaluator.args.get("external_server") and not evaluator.act_as_middlebox: + evaluator.stop_server(environment, server) + + return ind.environment_id, ind.fitness + + @staticmethod + def get_args(command): + """ + Defines required global args for all plugins + """ + parser = argparse.ArgumentParser(description='HTTP plugin runner', allow_abbrev=False) + parser.add_argument('--disable-port-negotiation', action='store_true', help="disables port negotiation between remote client and local server") + parser.add_argument('--use-external-sites', action='store_true', help="draw from the pool of external servers (defined in external_sites.py) for testing.") + parser.add_argument('--environment-id', action='store', help="ID of the current environment") + parser.add_argument('--output-directory', action='store', help="Where to output results") + parser.add_argument('--port', action='store', type=int, default=80, help='port to use') + args, _ = parser.parse_known_args(command) + return vars(args) + + +def check_censorship(site, evaluator, environment, logger): + """ + Make a request to the given site to test if it is censored. Used to test + a site for residual censorship before using it. + """ + command = "curl -s %s -m 5" % site + if environment.get("remote"): + stdout, stderr = evaluator.remote_exec_cmd(environment["remote"], command, logger, timeout=5) + for line in stderr: + if "Connection reset by peer" in line: + logger.info("Residual censorship detected on %s." % site) + return False + return True + try: + requests.get(site, allow_redirects=False, timeout=3) + return True + except (requests.exceptions.ConnectionError, + ConnectionResetError, + urllib.error.URLError, + requests.exceptions.Timeout, + Exception) as e: + logger.error("Could not reach site %s" % site) + return False + + +class TestServer(): + """ + Context manager to retrieve a test server from the external server pool. + """ + def __init__(self, requested_site, evaluator, environment, logger): + self.requested_site = requested_site + self.logger = logger + self.evaluator = evaluator + self.environment = environment + self.site_to_test = None + + def __enter__(self): + """ + Reserves a site for testing for this worker. + """ + if self.requested_site: + return self.requested_site + + while True: + site_to_test = TEST_SITES.pop(0) + current_seconds = calendar.timegm(time.gmtime()) + + # Check if our current time is at least JAIL_TIME away from the last time + # we tried to use this site. + if (current_seconds - JAIL_TRACKER[site_to_test]) > JAIL_TIME: + site_good = False + self.logger.debug("Checking %s for censorship." % site_to_test) + site_good = check_censorship(site_to_test, self.evaluator, self.environment, self.logger) + + if site_good: + self.logger.debug("Using site %s for testing." % site_to_test) + break + else: + self.logger.debug("Residual censorship detected for %s" % site_to_test) + + # If we didn't break, put the site back at the end of the list + TEST_SITES.append(site_to_test) + if self.logger: + self.logger.debug("%s is not yet available to test - only %d seconds have \ + transpired since last test - waiting 5 seconds." % + (site_to_test, current_seconds - JAIL_TRACKER[site_to_test])) + time.sleep(0.1) + + # Store the site we're testing to re-add it to the pool on exit + self.site_to_test = site_to_test + return site_to_test + + def __exit__(self, exc_type, exc_value, trace): + """ + Cleans up and returns the site in testing to the test pool. + """ + if self.site_to_test: + self.logger.debug("Returning %s to pool of sites." % self.site_to_test) + TEST_SITES.append(self.site_to_test) + JAIL_TRACKER[self.site_to_test] = calendar.timegm(time.gmtime()) + + # Pass through exceptions + if exc_type is not None: + traceback.print_exception(exc_type, exc_value, trace) + return False + return True + + +# Note that this code is not for debugging and cannot be removed - +# this is how the evaluator runs the client. +if __name__ == "__main__": + main(vars(get_args())) diff --git a/plugins/http/server.py b/plugins/http/server.py new file mode 100644 index 0000000..bf37732 --- /dev/null +++ b/plugins/http/server.py @@ -0,0 +1,69 @@ +import argparse +import logging +import os +import tempfile +import subprocess + +import actions.utils +from plugins.plugin_server import ServerPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class HTTPServer(ServerPlugin): + """ + Defines the HTTP client. + """ + name = "http" + def __init__(self, args): + """ + Initializes the HTTP client. + """ + ServerPlugin.__init__(self) + self.args = args + if args: + self.port = args["port"] + self.tmp_dir = None + + @staticmethod + def get_args(command): + """ + Defines arguments for this plugin + """ + super_args = ServerPlugin.get_args(command) + + parser = argparse.ArgumentParser(description='HTTP Server') + parser.add_argument('--port', action='store', default="", help='port to run this server on') + + args, _ = parser.parse_known_args(command) + args = vars(args) + super_args.update(args) + return super_args + + def run(self, args, logger): + """ + Initializes the HTTP server. + """ + # Create a temporary directory to run out of so we're not hosting files + self.tmp_dir = tempfile.TemporaryDirectory() + + # Default all output to /dev/null + stdout, stderr = subprocess.DEVNULL, subprocess.DEVNULL + + # If we're in debug mode, don't send output to /dev/null + if actions.utils.get_console_log_level() == "debug": + stdout, stderr = None, None + + # Start the server + try: + subprocess.check_call(["python3", "-m", "http.server", str(args.get('port'))], stderr=stderr, stdout=stdout, cwd=self.tmp_dir.name) + except subprocess.CalledProcessError as exc: + logger.debug("Server exited: %s", str(exc)) + + def stop(self): + """ + Stops this server. + """ + if self.tmp_dir: + self.tmp_dir.cleanup() + ServerPlugin.stop(self) diff --git a/plugins/plugin.py b/plugins/plugin.py new file mode 100644 index 0000000..ea95fe9 --- /dev/null +++ b/plugins/plugin.py @@ -0,0 +1,22 @@ +import argparse +import copy +import os +import logging +import subprocess +import sys + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(BASEPATH) + +import actions.sniffer +import actions.utils + + +class Plugin(): + """ + Defines superclass for application plugins. + """ + # Normal plugins evaluate strategies one by one for a clean slate. + # Plugins can override that behavior and evaluate the entire population pool + # at once with this flag. + override_evaluation = False diff --git a/plugins/plugin_client.py b/plugins/plugin_client.py new file mode 100644 index 0000000..9c1ff46 --- /dev/null +++ b/plugins/plugin_client.py @@ -0,0 +1,130 @@ +import argparse +import os +import sys +import time + +from scapy.all import send, IP, TCP, Raw + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(BASEPATH) +sys.path.append(PROJECT_ROOT) + +import actions.sniffer +import engine + +from plugins.plugin import Plugin + + +class ClientPlugin(Plugin): + """ + Defines superclass for each application plugin. + """ + def __init__(self): + self.enabled = True + + @staticmethod + def get_args(command): + """ + Defines required global args for all plugins + """ + # Do not add a help message; this allows us to collect the arguments from server plugins + parser = argparse.ArgumentParser(description='Client plugin runner', allow_abbrev=False, add_help=False) + parser.add_argument('--test-type', action='store', choices=actions.utils.get_plugins(), default="http", help="plugin to launch") + parser.add_argument('--environment-id', action='store', help="ID of the current environment") + parser.add_argument('--output-directory', action='store', help="Where to output results") + parser.add_argument('--no-engine', action="store_true", + help="Only run the test without the geneva engine") + parser.add_argument('--server-side', action="store_true", help="run the Geneva engine on the server side, not the client") + parser.add_argument('--strategy', action='store', default="", help='strategy to run') + parser.add_argument('--server', action='store', help="server to connect to") + parser.add_argument('--log', action='store', default="debug", + choices=("debug", "info", "warning", "critical", "error"), + help="Sets the log level") + parser.add_argument('--port', action='store', type=int, help='port to run this server on') + + parser.add_argument('--wait-for-censor', action='store_true', help='send control packets to the censor to get startup confirmation') + + parser.add_argument('--bad-word', action='store', help="forbidden word to test with", default="ultrasurf") + + args, _ = parser.parse_known_args(command) + return vars(args) + + def start(self, args, logger): + """ + Runs this plugin. + """ + logger.debug("Launching %s" % self.name) + fitness = -1000 + + output_path = os.path.join(PROJECT_ROOT, args.get("output_directory")) + eid = args.get("environment_id") + use_engine = not args.get("no_engine") + port = args.get("port") + server_side = args.get("server_side") + assert port, "Need to specify a port in order to launch a sniffer" + + pcap_filename = os.path.join(output_path, "packets", eid + "_client.pcap") + + # Start a sniffer to capture traffic that the plugin generates + with actions.sniffer.Sniffer(pcap_filename, port, logger) as sniff: + + # Conditionally initialize the engine + with engine.Engine(port, args.get("strategy"), server_side=False, environment_id=eid, output_directory=output_path, log_level=args.get("log", "info"), enabled=use_engine) as eng: + # Wait for the censor to start up, if one is running + if args.get("wait_for_censor"): + self.wait_for_censor(args.get("server"), port, eid, output_path) + + # Run the plugin + fitness = self.run(args, logger, engine=eng) + logger.debug("Plugin client has finished.") + if use_engine: + fitness = actions.utils.punish_fitness(fitness, logger, eng) + + # If fitness files are disabled, just return + if args.get("no_fitness_file"): + return fitness + + logger.debug("Fitness: %d", fitness) + + actions.utils.write_fitness(fitness, output_path, eid) + return fitness + + def wait_for_censor(self, serverip, port, environment_id, log_dir): + """ + Sends control packets to the censor for up to 20 seconds until it's ready. + """ + for _ in range(0, 200): + check = IP(dst=serverip)/TCP(dport=int(port), sport=2222, seq=13337)/Raw(load="checking") + send(check, verbose=False) + ready_path = os.path.join(BASEPATH, log_dir, actions.utils.FLAGFOLDER, "%s.censor_ready" % environment_id) + if os.path.exists(ready_path): + os.system("rm %s" % ready_path) + break + time.sleep(0.1) + else: + return False + return True + + +def main(command): + """ + Used to invoke the plugin client from the command line. + """ + # Must use the superclasses arg parsing first to figure out the plugin to use + plugin = ClientPlugin.get_args(command)["test_type"] + + # Import that plugin + _, cls = actions.utils.import_plugin(plugin, "client") + + # Ask the plugin to parse the args + plugin_args = cls.get_args(command) + + # Instantiate the plugin + client_plugin = cls(plugin_args) + + # Define a logger and launch the plugin + with actions.utils.Logger(plugin_args["output_directory"], __name__, "client", plugin_args["environment_id"], log_level=plugin_args.get("log")) as logger: + client_plugin.start(plugin_args, logger) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/plugins/plugin_server.py b/plugins/plugin_server.py new file mode 100644 index 0000000..0cd2e86 --- /dev/null +++ b/plugins/plugin_server.py @@ -0,0 +1,207 @@ +import argparse +import threading +import multiprocessing +import os +import psutil +import socket +import sys +import time + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(BASEPATH) +sys.path.append(PROJECT_ROOT) + +import actions.sniffer +import engine + +from plugins.plugin import Plugin + + +class ServerPlugin(Plugin): + """ + Defines superclass for each application plugin. + """ + def __init__(self): + self.enabled = True + self.server_proc = None + self.sniffer = None + self.engine = None + + @staticmethod + def get_args(command): + """ + Defines required global args for all plugins + """ + # Do not add a help message; this allows us to collect the arguments from server plugins + parser = argparse.ArgumentParser(description='Server plugin runner', allow_abbrev=False, add_help=False) + parser.add_argument('--test-type', action='store', choices=actions.utils.get_plugins(), default="http", help="plugin to launch") + parser.add_argument('--environment-id', action='store', help="ID of the current environment") + parser.add_argument('--output-directory', action='store', help="Where to output results") + parser.add_argument('--no-engine', action="store_true", + help="Only run the test without the geneva engine") + parser.add_argument('--server-side', action="store_true", help="run the Geneva engine on the server side, not the client") + parser.add_argument('--strategy', action='store', default="", help='strategy to run') + parser.add_argument('--log', action='store', default="debug", + choices=("debug", "info", "warning", "critical", "error"), + help="Sets the log level") + parser.add_argument('--port', action='store', type=int, help='port to run this server on') + + parser.add_argument('--external-server', action='store_true', help="use an external server for testing.") + + parser.add_argument('--sender-ip', action='store', help="IP address of sending machine, used for NAT") + parser.add_argument('--forward-ip', action='store', help="IP address to forward traffic to") + parser.add_argument('--routing-ip', action='store', help="routing IP for this computer for server-side evaluation.") + parser.add_argument('--public-ip', action='store', help="public facing IP for this computer for server-side evaluation.") + + + parser.add_argument('--no-wait-for-server', action='store_true', help="disable blocking until the server is bound on a given port") + parser.add_argument('--wait-for-shutdown', action='store_true', help="monitor for the .shutdown_server flag to shutdown this server.") + + args, _ = parser.parse_known_args(command) + return vars(args) + + def start(self, args, logger): + """ + Runs this plugin. + """ + logger.debug("Launching %s server" % self.name) + + output_path = os.path.join(PROJECT_ROOT, args["output_directory"]) + eid = args["environment_id"] + use_engine = not args.get("no_engine", False) + port = args["port"] + server_side = args["server_side"] + log_level = args["log"] + strategy = args.get("strategy", "") + assert port, "Need to specify a port in order to launch a sniffer" + forwarder = {} + # If NAT options were specified to train as a middle box, set up the engine's + # NAT configuration + if args.get("sender_ip"): + assert args.get("forward_ip") + assert args.get("sender_ip") + assert args.get("routing_ip") + forwarder["forward_ip"] = args["forward_ip"] + forwarder["sender_ip"] = args["sender_ip"] + forwarder["routing_ip"] = args["routing_ip"] + + pcap_filename = os.path.join(output_path, "packets", eid + "_server.pcap") + + # We cannot use the context managers as normal here, as this method must return and let the evaluator continue + # doing its thing. If we used the context managers, they would be cleaned up on method exit. + + # Start a sniffer to capture traffic that the plugin generates + self.sniffer = actions.sniffer.Sniffer(pcap_filename, int(port), logger).__enter__() + + # Conditionally initialize the engine + self.engine = engine.Engine(port, strategy, server_side=True, environment_id=eid, output_directory=output_path, log_level=args.get("log", "info"), enabled=use_engine, forwarder=forwarder).__enter__() + + # Run the plugin + self.server_proc = multiprocessing.Process(target=self.start_thread, args=(args, logger)) + self.server_proc.start() + + # Create a thread to monitor if we need to + if args.get("wait_for_shutdown"): + threading.Thread(target=self.wait_for_shutdown, args=(args, logger)).start() + + # Shortcut wait for server if a plugin has disabled it + if args.get("no_wait_for_server"): + return + + # Block until the server has started up + self.wait_for_server(args, logger) + + def start_thread(self, args, logger): + """ + Calls the given run function, designed to be run in a separate process. + """ + self.run(args, logger) + + def wait_for_server(self, args, logger): + """ + Waits for server to startup - returns when the server port is bound to by the server. + """ + logger.debug("Monitoring for server startup on port %s" % args["port"]) + max_wait = 30 + count = 0 + while count < max_wait: + if count % 10 == 0: + logger.debug("Waiting for server port binding") + # Bind TCP socket + try: + with socket.socket() as sock: + sock.bind(('', int(args["port"]))) + except OSError: + break + time.sleep(0.5) + count += 1 + else: + logger.warn("Server never seemed to bind to port") + return + + self.write_startup_file(args, logger) + + def write_startup_file(self, args, logger): + """ + Writes a flag file to disk to signal to the evaluator it has started up + """ + # Touch a file to tell the evaluator we are ready + flag_file = os.path.join(PROJECT_ROOT, args["output_directory"], "flags", "%s.server_ready" % args["environment_id"]) + open(flag_file, "a").close() + logger.debug("Server ready.") + + def wait_for_shutdown(self, args, logger): + """ + Checks for the .server_shutdown flag to shutdown this server. + """ + flag_file = os.path.join(PROJECT_ROOT, args["output_directory"], "flags", "%s.server_shutdown" % args["environment_id"]) + while True: + if os.path.exists(flag_file): + break + time.sleep(0.5) + logger.debug("Server for %s shutting down." % args["environment_id"]) + self.stop() + logger.debug("Server %s stopped." % args["environment_id"]) + + def stop(self): + """ + Terminates the given process. + """ + self.engine.__exit__(None, None, None) + self.sniffer.__exit__(None, None, None) + # In order to clean up all the child processes a server may have started, + # iterate over all of the process children and terminate them + proc = psutil.Process(self.server_proc.pid) + for child in proc.children(recursive=True): + child.terminate() + proc.terminate() + + def punish_fitness(self, fitness, logger): + """ + Punish fitness. + """ + return actions.utils.punish_fitness(fitness, logger, self.engine) + + +def main(command): + """ + Used to invoke the server plugin from the command line. + """ + # Must use the superclasses arg parsing first to figure out the plugin to use + plugin = ServerPlugin.get_args(command)["test_type"] + + # Import that plugin + mod, cls = actions.utils.import_plugin(plugin, "server") + + # Ask the plugin to parse the args + plugin_args = cls.get_args(command) + + # Instantiate the plugin + server_plugin = cls(plugin_args) + + # Define a logger and launch the plugin + with actions.utils.Logger(plugin_args["output_directory"], __name__, "server", plugin_args["environment_id"], log_level=plugin_args["log"]) as logger: + server_plugin.start(plugin_args, logger) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/plugins/sni/client.py b/plugins/sni/client.py new file mode 100644 index 0000000..77d7ef1 --- /dev/null +++ b/plugins/sni/client.py @@ -0,0 +1,89 @@ +""" +Runs an SNI request, confirms the connection was not torn down +""" + +import argparse +import logging +import os +import random +import socket +import subprocess as sp +import sys +import time +import traceback +import urllib.request + +import requests + +socket.setdefaulttimeout(1) + +import external_sites +import actions.utils + +from plugins.plugin_client import ClientPlugin + +BASEPATH = os.path.dirname(os.path.abspath(__file__)) + + +class SNIClient(ClientPlugin): + """ + Defines the SNI client. + """ + name = "sni" + + def __init__(self, args): + """ + Initializes the sni client. + """ + ClientPlugin.__init__(self) + self.args = args + + @staticmethod + def get_args(command): + """ + Defines required args for this plugin + """ + super_args = ClientPlugin.get_args(command) + parser = argparse.ArgumentParser(description='HTTP Client') + + parser.add_argument('--server', action='store', default="www.wikipedia.org", help='SNI request to make') + parser.add_argument('--injected-cert-contains', action='store', help='text that injected cert will contain') + parser.add_argument('--ip', action='store', help='IP address to send the request to') + + args, _ = parser.parse_known_args(command) + args = vars(args) + + super_args.update(args) + return super_args + + def run(self, args, logger, engine=None): + """ + Try to make a forbidden SNI request to the server. + """ + fitness = 0 + output = "" + injected_cert_contains = args.get("injected_cert_contains", "") + try: + server = args.get("server", "www.wikipedia.org") + ip = args.get("ip", "") + cmd = "curl -v --resolve '%s:443:%s' ::%s: https://%s" % (server, ip, server, server) + logger.debug(cmd) + output = sp.check_output(cmd, timeout=8, shell=True, stderr=sp.STDOUT) + logger.debug(output) + except sp.CalledProcessError as exc: + logger.debug(exc.output) + if b"connection reset" in exc.output: + fitness = -360 + else: + fitness = -400 + except sp.TimeoutExpired: + logger.debug("Client timed out") + fitness = -400 + else: + logger.debug(output) + # Check for known signature of the injected certificate + if injected_cert_contains and injected_cert_contains in output: + fitness = -360 + else: + fitness = 400 + return fitness diff --git a/requirements.txt b/requirements.txt index 950f099..c296fe4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,22 @@ +pytest scapy==2.4.3 requests +dnspython +docker +anytree +graphviz netifaces netfilterqueue cryptography==2.8 +paramiko +codecov +pytest-cov +dnspython +tld +python-dotenv +mysql-connector +slackclient==1.3.1 # have not yet ported to support 2.x +beautifulsoup4 requests -anytree +psutil +tqdm diff --git a/tests/DNS/zones/example.com b/tests/DNS/zones/example.com new file mode 100644 index 0000000..aa3d185 --- /dev/null +++ b/tests/DNS/zones/example.com @@ -0,0 +1,26 @@ +$TTL 36000 +example.com. IN SOA ns1.example.com. hostmaster.example.com. ( + 2005081201 ; serial + 28800 ; refresh (8 hours) + 1800 ; retry (30 mins) + 2592000 ; expire (30 days) + 86400 ) ; minimum (1 day) + +example.com. 86400 NS ns1.example.com. +example.com. 86400 NS ns2.example.com. +example.com. 86400 MX 10 mail1.n2.example.com. +example.com. 86400 MX 20 mail2.example.com. +example.com. 86400 A 192.168.10.10 +example.com. 86400 A 192.168.10.11 +example.com. 86400 TXT "v=spf1 a:mail.example.com -all" + +ns1.example.com. 86400 A 192.168.1.10 +ns1.example.com. 86400 A 192.168.1.11 +ns2.example.com. 86400 A 192.168.1.20 +mail.example.com. 86400 A 192.168.2.10 +mail2.example.com. 86400 A 192.168.2.20 +www2.example.com. 86400 A 192.168.10.20 + +www.example.com. 86400 CNAME example.com. +ftp.example.com. 86400 CNAME example.com. +webmail.example.com. 86400 CNAME example.com. \ No newline at end of file diff --git a/tests/DNS/zones/example2.com b/tests/DNS/zones/example2.com new file mode 100644 index 0000000..213b416 --- /dev/null +++ b/tests/DNS/zones/example2.com @@ -0,0 +1,26 @@ +$TTL 36000 +example2.com. IN SOA ns1.example2.com. hostmaster.example2.com. ( + 2005081201 ; serial + 28800 ; refresh (8 hours) + 1800 ; retry (30 mins) + 2592000 ; expire (30 days) + 86400 ) ; minimum (1 day) + +example2.com. 86400 NS ns1.example2.com. +example2.com. 86400 NS ns2.example2.com. +example2.com. 86400 MX 10 mail1.n2.example2.com. +example2.com. 86400 MX 20 mail2.example2.com. +example2.com. 86400 A 192.168.111.10 +example2.com. 86400 A 192.168.111.11 +example2.com. 86400 TXT "v=spf1 a:mail.example2.com -all" + +ns1.example2.com. 86400 A 192.168.110.10 +ns1.example2.com. 86400 A 192.168.110.11 +ns2.example2.com. 86400 A 192.168.110.20 +mail.example2.com. 86400 A 192.168.120.10 +mail2.example2.com. 86400 A 192.168.120.20 +www2.example2.com. 86400 A 192.168.100.20 + +www.example2.com. 86400 CNAME example2.com. +ftp.example2.com. 86400 CNAME example2.com. +webmail.example2.com. 86400 CNAME example2.com. \ No newline at end of file diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..45c2c48 --- /dev/null +++ b/tests/common.py @@ -0,0 +1,84 @@ +import subprocess +import sys +# Include the root of the project +sys.path.append("..") + +import actions.strategy +import actions.utils +import evolve +import evaluator + +import pytest + + +def run_test(logger, solution, censor, test_type, log_on_success=False, log_on_fail=False): + """ + Tests a given solution against a given censor under a given test type + using the given log level. + """ + # Test if docker is running + try: + subprocess.check_output(['docker', 'ps']) + except subprocess.CalledProcessError: + pytest.fail("Docker is not running") + + try: + # Parse the string representation of the solution + strat = actions.utils.parse(solution, logger) + logger.info("Parsed strategy %s" % (str(strat))) + + # Confirm the parsing was correct + assert str(strat).strip() == solution, "Failed to correctly parse given strategy" + + logger.info("Testing %s" % censor) + + # Setup the external server to test with, if an http test is done + if test_type == "echo": + external_server = None + elif test_type == "http": + external_server = "facebook.com" + + # Create an evaluator + cmd = [ + "--test-type", "echo", + "--censor", censor, + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + if log_on_success: + cmd += ["--log-on-success"] + if log_on_fail: + cmd += ["--log-on-fail"] + tester = evaluator.Evaluator(cmd, logger) + + # Use the fitness function to evaluate the strategy + population = evolve.fitness_function(logger, [strat], tester) + + # Check that we got back the same number of individuals we gave + assert len(population) == 1, "Population size changed" + + # Shutdown the evaluator + tester.shutdown() + + # Retrieve the fitness from the individual + return population[0].fitness + finally: + clean_containers() + + +def clean_containers(): + """ + Cleans up the client_main, censor_main, and server_main containers used by the evaluator + if it fails to. + """ + for name in ["client", "censor", "server"]: + try: + subprocess.check_output(["docker", "stop", "%s_main" % name], stderr=subprocess.PIPE) + except subprocess.CalledProcessError: + pass + try: + subprocess.check_output(["docker", "rm", "%s_main"], stderr=subprocess.PIPE) + except subprocess.CalledProcessError: + pass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4754185 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,98 @@ +import json +import pytest +import logging +import subprocess +import tempfile +import os +import shutil + +import evolve +import actions.packet + + +def pytest_addoption(parser): + """ + Adds options to pytest + """ + parser.addoption( + "--evolve-logger", action="store", choices=("debug", "info", "warning", "critical", "error"), help="Sets the log level", default="info" + ) + + +@pytest.fixture(scope="session") +def logger(request): + """ + Returns log level requested. + """ + # On some systems, docker and urllib3 log levels are cleared (such as Travis) + # Reset them to warnings only to keep logging sane. + logging.getLogger("docker").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + level = request.config.getoption("--evolve-logger") + logger = evolve.setup_logger(level) + return logger + + +@pytest.fixture(autouse=True, scope="function") +def reset_packet_restrictions(): + """ + Autouse feature to make sure tests have a clean slate for processing. + """ + actions.packet.Packet.reset_restrictions() + + +@pytest.mark.tryfirst +@pytest.fixture(autouse=True, scope="function") +def blank_new_line(): + """ + Autouse feature to print a new line after the test name for cleaner printing. + """ + print("") + + +@pytest.fixture +def client_worker(request): + """ + Defines a client worker fixture. This creates a docker container + with SSH enabled and the code mounted to test the evaluator's client workers. + """ + container = {} + worker_path = tempfile.mkdtemp() + os.makedirs(os.path.join(worker_path, "test_container")) + info_path = os.path.join(worker_path, "worker.json") + worker_dict = { + "ip" : "127.0.0.1", + "port" : 2222, + "username" : "root", + "password" : "Docker!", + "geneva_path" : "/code", + "python" : "python3" + } + + with open(info_path, "w") as fd: + json.dump(worker_dict, fd) + container["worker"] = info_path + + def fin(cid): + shutil.rmtree(worker_path) + if cid: + print("\nCleaning up container") + subprocess.check_call(["docker", "stop", cid]) + + cid = None + # Run the base docker container to give us a worker client + cid = subprocess.check_output(["docker", "run", "--privileged", "--dns=8.8.8.8", "-id", "-p", "2222:22", "-v", "%s:/code" % os.path.abspath(os.getcwd()), "base"]).decode("utf-8").strip() + request.addfinalizer(lambda: fin(cid)) + print("\nCreated container %s" % cid[:8]) + container["id"] = cid + container["cid"] = cid[:8] + + #output = subprocess.check_output(["docker", "exec", "-i", cid, "ifconfig", "eth0"]) + + #ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', output.decode("utf-8"))[0] + #print("Parsed container ip: %s" % ip) + container["ip"] = "0.0.0.0" + + subprocess.check_call(["docker", "exec", "-i", cid, "service", "ssh", "start"]) + return container diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..1ceab94 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -p no:warnings diff --git a/tests/test_censors.py b/tests/test_censors.py new file mode 100644 index 0000000..cff6f95 --- /dev/null +++ b/tests/test_censors.py @@ -0,0 +1,79 @@ +import logging +import pytest +import sys +# Include the root of the project +sys.path.append("..") + +import censors.censor_driver +import library +import common + + +def get_censors(): + """ + Retrieves a list of all available censors for testing. + """ + tests = [] + docker_censors = censors.censor_driver.get_censors() + test_type = "echo" + for censor in docker_censors.keys(): + # Skip the dummy censor + if censor == "dummy": + continue + tests.append((censor, test_type)) + return tests + + +def get_tests(): + """ + Returns a list of tuples of tests of combinations of solutions and censors. + """ + tests = [] + all_censors = censors.censor_driver.get_censors().keys() + test_type = "echo" + for solution in library.LAB_STRATEGIES: + # Calculate the set of censors a solution does not beat + defeated = set(all_censors) - set(solution["censors"]) + for censor in defeated: + tests.append((solution["strategy"], censor, test_type)) + + return tests + + +@pytest.mark.parametrize("censor, test_type", get_censors()) +def test_censors(logger, censor, test_type): + """ + Tests each censor against an empty strategy to confirm the censor works. + """ + # Test each censor against an empty strategy. + # \/ is an empty strategy, representing no input or output chains + test_library(logger, "\/", censor, test_type) + + +@pytest.mark.skip() +@pytest.mark.parametrize("solution, censor, test_type", get_tests()) +def test_library(logger, solution, censor, test_type): + """ + Pulls each solution from the solution library and tests it against + it's corresponding censor to confirm the censor wins. + """ + if censor == "dummy": + pytest.skip("Skipping dummy censor test.") + fitness = common.run_test(logger, solution, censor, test_type, log_on_success=True) + # If the fitness was less than 0, the strategy failed to beat the censor + if fitness > 0: + pytest.fail("Fitness was %d - strategy beat censor." % fitness) + + +def test_one_library(logger): + """ + Runs a test using one solution from the library as a quick litmus + test for code health. + """ + solution = "\/ [TCP:dataofs:5]-drop-|" + censor = "censor2" + test_type = "echo" + fitness = common.run_test(logger, solution, censor, test_type, log_on_fail=True) + # If the fitness was less than 0, the strategy failed to beat the censor + if fitness < 0: + pytest.fail("Fitness was %d - censor beat strategy." % fitness) diff --git a/tests/test_compress.py b/tests/test_compress.py new file mode 100644 index 0000000..7cddaf3 --- /dev/null +++ b/tests/test_compress.py @@ -0,0 +1,26 @@ +import logging +import os +import pytest +import tempfile + +import actions.tree +import actions.drop +import actions.tamper +import actions.duplicate +import actions.utils +import actions.strategy +import engine +import evaluator +import evolve + +import netifaces +from scapy.all import IP, TCP + + +def test_compression_strategy(logger): + """ + Tests dns compression strategy. + """ + with engine.Engine(53, "[UDP:dport:53]-tamper{DNS:qd:compress}-|", server_side=False, environment_id="compress_test", output_directory=actions.utils.RUN_DIRECTORY, log_level=actions.utils.CONSOLE_LOG_LEVEL): + os.system("dig @8.8.8.8 google.com") + diff --git a/tests/test_dns_server.py b/tests/test_dns_server.py new file mode 100644 index 0000000..4d2d0aa --- /dev/null +++ b/tests/test_dns_server.py @@ -0,0 +1,443 @@ +# Scapy modules +from scapy.layers.dns import IP, UDP, raw, DNS as DNS_, DNSQR, struct + +# DNS Modules +import dns.zone + +# Import the root of the project: used to import DNSServer +import os +import sys +import inspect +import logging + +import pytest + +basepath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parent_dir = os.path.dirname(basepath) +sys.path.insert(0, parent_dir) + +import evolve +from plugins.dns.server import DNSServer + +# Default values +INTERFACE = 'lo' +LISTENER = DNSServer.socket_UDP +PORT = 53 +AUTHORITY = False +DNS_RESOLVER = "1.1.1.1" +LOG_DIR = basepath + "/DNS/" +ZONES_DIR = basepath + "/DNS/zones/" +LOGGING_LEVEL = logging.INFO + +# Error definitions +RECORD_COUNT_ERROR = "record_count_error" +RECORD_VALUE_ERROR = "record_value_error" + + +@pytest.mark.skip() +@pytest.mark.parametrize("listener", [DNSServer.socket_UDP, DNSServer.socket_TCP, DNSServer.netfilter_queue]) +def test_dns_server(listener, logger): + """ + Tests the main method + """ + # TODO test is currently disabled, will be replaced by a test that + # tests the full functionality of receiving DNS queries + args = { + 'interface': INTERFACE, + 'port': PORT, + 'authority': AUTHORITY, + 'zones_dir': ZONES_DIR, + 'log_dir': LOG_DIR, + 'dry_run': True, + 'listener': listener + } + + server = DNSServer.server.main(args) + + +@pytest.mark.parametrize("listener", [DNSServer.socket_UDP, DNSServer.socket_TCP, DNSServer.netfilter_queue]) +def test_zone_records(listener, logger): + """ + Tests if it can read the information in the zones file correctly + """ + args = { + "interface": INTERFACE, + "listener": listener, + "port": PORT, + "authority": AUTHORITY, + "dns_resolver": DNS_RESOLVER, + "zones_dir": ZONES_DIR + } + + # Testing variable + server = DNSServer(args, logger=logger) + + server.load_zones() + + # Answer variables + example_com = dns.zone.from_file(ZONES_DIR + "example.com", "example.com", rdclass=1, relativize=False) + example2_com = dns.zone.from_file(ZONES_DIR + "example2.com", "example2.com", rdclass=1, relativize=False) + + # ---------------- Testing A records ----------------- + # No errors + check_records(server, example_com, "example.com.", "A") + check_records(server, example_com, "ns1.example.com.", "A") + check_records(server, example_com, "ns2.example.com.", "A") + check_records(server, example_com, "mail.example.com.", "A") + check_records(server, example_com, "mail2.example.com.", "A") + check_records(server, example_com, "www2.example.com.", "A") + + # Errors + # ns1.example.com. has 2 A records while ns2.example.com. has 1 A record + check_records(server, example_com, "ns2.example.com.", "A", False, RECORD_COUNT_ERROR, "ns1.example.com.") + # Both example.com. and ns1.example.com. have 2 A records but the value of those records are different + check_records(server, example_com, "example.com.", "A", False, RECORD_VALUE_ERROR, "ns1.example.com.") + + # No errors with a different zone file + check_records(server, example2_com, "example2.com.", "A") + check_records(server, example2_com, "ns1.example2.com.", "A") + check_records(server, example2_com, "ns2.example2.com.", "A") + check_records(server, example2_com, "mail.example2.com.", "A") + check_records(server, example2_com, "mail2.example2.com.", "A") + check_records(server, example2_com, "www2.example2.com.", "A") + + # Errors with a different zone + # ns1.example.com. has 2 A records while ns2.example.com. has 1 A record + check_records(server, example2_com, "ns2.example2.com.", "A", False, RECORD_COUNT_ERROR, "ns1.example2.com.") + # Both example.com. and ns1.example.com. have 2 A records but the value of those records are different + check_records(server, example2_com, "example2.com.", "A", False, RECORD_VALUE_ERROR, "ns1.example2.com.") + + # ---------------- Testing TXT records ----------------- + # No errors + check_records(server, example_com, "example.com.", "TXT") + check_records(server, example2_com, "example2.com.", "TXT") + + # ---------------- Testing MX records ----------------- + # No errors + check_records(server, example_com, "example.com.", "MX") + check_records(server, example2_com, "example2.com.", "MX") + + # ---------------- Testing NS records ----------------- + # No errors + check_records(server, example_com, "example.com.", "NS") + check_records(server, example2_com, "example2.com.", "NS") + + # ---------------- Testing CNAME records ----------------- + # No errors + check_records(server, example_com, "www.example.com.", "CNAME") + check_records(server, example2_com, "www.example2.com.", "CNAME") + + # ---------------- Testing NXDOMAIN ----------------- + # No errors + check_nxdomain(server, "www3.example.com.", "A") + check_nxdomain(server, "www3.example.com.", "TXT") + check_nxdomain(server, "www3.example.com.", "NS") + check_nxdomain(server, "www3.example.com.", "MX") + check_nxdomain(server, "www3.example.com.", "CNAME") + +def test_forwarding(logger): + """ + Tests if DNSServer properly enables and disables forwarding of DNS queries that it does not have answers to + """ + args = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": AUTHORITY, + "dns_resolver": DNS_RESOLVER, + "zones_dir": ZONES_DIR + } + + args_no_forward = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": AUTHORITY, + "dns_resolver": None, + "zones_dir": ZONES_DIR + } + + # Testing variable + server = DNSServer(args, logger=logger) + server_no_forward = DNSServer(args_no_forward, logger=logger) + + # Zone loading happens during actual startup, so load it here + server.load_zones() + server_no_forward.load_zones() + + # Answer variables + example_com = dns.zone.from_file(ZONES_DIR + "example.com", "example.com", rdclass=1, relativize=False) + example2_com = dns.zone.from_file(ZONES_DIR + "example2.com", "example2.com", rdclass=1, relativize=False) + + # Test if it can forward a query + check_record_exists(server, "google.com.", "A") + check_record_exists(server, "msn.com.", "A") + + # ------------- NXDOMAIN --------------- + # NXDOMAIN for all domains outside of the zones configured + check_nxdomain(server_no_forward, "google.com.", "A") + check_nxdomain(server_no_forward, "google.com.", "TXT") + check_nxdomain(server_no_forward, "google.com.", "NS") + check_nxdomain(server_no_forward, "google.com.", "MX") + check_nxdomain(server_no_forward, "google.com.", "CNAME") + + check_nxdomain(server_no_forward, "msn.com.", "A") + check_nxdomain(server_no_forward, "msn.com.", "TXT") + check_nxdomain(server_no_forward, "msn.com.", "NS") + check_nxdomain(server_no_forward, "msn.com.", "MX") + check_nxdomain(server_no_forward, "msn.com.", "CNAME") + + # NXDOMAIN for domains declared in the zones but does not exist + check_nxdomain(server_no_forward, "www3.example.com.", "A") + check_nxdomain(server_no_forward, "www3.example.com.", "TXT") + check_nxdomain(server_no_forward, "www3.example.com.", "NS") + check_nxdomain(server_no_forward, "www3.example.com.", "MX") + check_nxdomain(server_no_forward, "www3.example.com.", "CNAME") + + # ------------- Resource Records --------------- + # Resource Records declared in the zones + + check_records(server_no_forward, example_com, "example.com.", "A") + check_records(server_no_forward, example_com, "example.com.", "TXT") + check_records(server_no_forward, example_com, "example.com.", "MX") + check_records(server_no_forward, example_com, "example.com.", "NS") + check_records(server_no_forward, example_com, "www.example.com.", "CNAME") + + check_records(server_no_forward, example2_com, "example2.com.", "A") + check_records(server_no_forward, example2_com, "example2.com.", "TXT") + check_records(server_no_forward, example2_com, "example2.com.", "MX") + check_records(server_no_forward, example2_com, "example2.com.", "NS") + check_records(server_no_forward, example2_com, "www.example2.com.", "CNAME") + + +def test_authority_reply(logger): + """ + Tests that the DNS responses correctly include the authority flag when set + """ + args = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": True, + "dns_resolver": DNS_RESOLVER, + "zones_dir": ZONES_DIR + } + + args_no_auth = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": False, + "dns_resolver": DNS_RESOLVER, + "zones_dir": ZONES_DIR + } + + server = DNSServer(args, logger=logger) + server_no_auth = DNSServer(args_no_auth, logger=logger) + + # Zone loading happens during actual startup, so load it here + server.load_zones() + server_no_auth.load_zones() + + example_com = dns.zone.from_file(ZONES_DIR + "example.com", "example.com", rdclass=1, relativize=False) + + # Test with authority - Zones configuration + check_records(server, example_com, "example.com.", "A", authority=True) + check_records(server, example_com, "example.com.", "TXT", authority=True) + check_records(server, example_com, "example.com.", "MX", authority=True) + check_records(server, example_com, "example.com.", "NS", authority=True) + check_records(server, example_com, "www.example.com.", "CNAME", authority=True) + + # Test with no authority - Zone configuration + check_records(server_no_auth, example_com, "example.com.", "A", authority=False) + check_records(server_no_auth, example_com, "example.com.", "TXT", authority=False) + check_records(server_no_auth, example_com, "example.com.", "MX", authority=False) + check_records(server_no_auth, example_com, "example.com.", "NS", authority=False) + check_records(server_no_auth, example_com, "www.example.com.", "CNAME", authority=False) + + # Test with authority - Zone configuration - NXDOMAIN + check_nxdomain(server, "www3.example.com.", "A", authority=True) + check_nxdomain(server, "www3.example.com.", "TXT", authority=True) + check_nxdomain(server, "www3.example.com.", "NS", authority=True) + check_nxdomain(server, "www3.example.com.", "MX", authority=True) + check_nxdomain(server, "www3.example.com.", "CNAME", authority=True) + + # Test without authority - Zone configuration - NXDOMAIN + check_nxdomain(server_no_auth, "www3.example.com.", "A", authority=False) + check_nxdomain(server_no_auth, "www3.example.com.", "TXT", authority=False) + check_nxdomain(server_no_auth, "www3.example.com.", "NS", authority=False) + check_nxdomain(server_no_auth, "www3.example.com.", "MX", authority=False) + check_nxdomain(server_no_auth, "www3.example.com.", "CNAME", authority=False) + + # Test with authority - DNS Forwarding - Exists + check_record_exists(server, "google.com.", "A", authority=True) + check_record_exists(server, "msn.com.", "A", authority=True) + + # Test without authority - DNS Forwarding - Exists + check_record_exists(server_no_auth, "google.com.", "A", authority=False) + check_record_exists(server_no_auth, "msn.com.", "A", authority=False) + + # Test with authority - DNS Forwarding - NXDOMAIN + check_nxdomain(server, "12398.google.com.", "A", authority=True) + check_record_exists(server, "12398.msn.com.", "A", authority=True) + + # Test without authority - DNS Forwarding - NXDOMAIN + check_nxdomain(server_no_auth, "12398.google.com.", "A", authority=False) + check_record_exists(server_no_auth, "12398.msn.com.", "A", authority=False) + + +def test_tld_does_not_exist(logger): + """ + Tests that if one queries for a TLD that does not exist, the program will simply respond with NXDOMAIN + :return: + """ + args = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": AUTHORITY, + "dns_resolver": DNS_RESOLVER, + "zones_dir": ZONES_DIR + } + + args_no_auth = { + "interface": INTERFACE, + "listener": LISTENER, + "port": PORT, + "authority": AUTHORITY, + "dns_resolver": None, + "zones_dir": ZONES_DIR + } + + server = DNSServer(args, logger=logger) + server_no_forward = DNSServer(args_no_auth, logger=logger) + + # Zone loading happens during actual startup, so load it here + server.load_zones() + server_no_forward.load_zones() + + check_nxdomain(server_no_forward, "google.tp.", "A") + check_nxdomain(server_no_forward, "google.techn.", "CNAME") + check_nxdomain(server_no_forward, "google.techno.", "MX") + check_nxdomain(server_no_forward, "google.technol.", "TXT") + check_nxdomain(server_no_forward, "google.technolo.", "NS") + + check_nxdomain(server, "google.tp.", "A") + check_nxdomain(server, "google.techn.", "CNAME") + check_nxdomain(server, "google.techno.", "MX") + check_nxdomain(server, "google.technol.", "TXT") + check_nxdomain(server, "google.technolo.", "NS") + + +def check_nxdomain(server, query, query_type, authority=False): + """ + Tests that the DNS response marks the query as NXDOMAIN + """ + + dns_query = IP(dst="127.0.0.1") / UDP(dport=53) / \ + DNS_(rd=1, qd=DNSQR(qname=query, qtype=query_type)) + dns_query = IP(raw(dns_query)) + + response = server.build_dns_response(dns_query) + + assert response[DNS_].rcode == 3 + assert response[DNS_].ancount == 0 + + if authority is True: + assert response[DNS_].aa == 1 + + +def get_value(record, query_type): + """ + Gets the value (rdata) of a specific resource record + """ + + if query_type == "TXT": + return dns.rdata._escapify(record.strings[0]) + elif query_type == "MX": + return (struct.pack("!H", record.preference) + record.exchange.to_wire(None, None)).decode('utf-8') + + return record.to_text() + + +def check_record_exists(server, query, query_type, authority=False): + """ + Checks if there is at least one resource record. + Optionally, check if the DNS response has the "Authoritative Answer" flag set + """ + dns_query = IP(dst="127.0.0.1") / UDP(dport=53) / \ + DNS_(rd=1, qd=DNSQR(qname=query, qtype=query_type)) + dns_query = IP(raw(dns_query)) + + response = server.build_dns_response(dns_query) + + assert response[DNS_].rcode == 0 + assert response[DNS_].ancount > 0 + + assert response[DNS_].an[0].rdata != '' + + if authority is True: + assert response[DNS_].aa == 1 + + +def check_records(server, answer, query, query_type, authority=False, error=None, other_query=None): + """ + Checks that the A record value & record count matches (if error is None) + Otherwise, if error is specified, then it checks to make sure that the error is achieved + Optionally, check if the DNS response has the "Authoritative Answer" flag set + """ + + dns_query = IP(dst="127.0.0.1") / UDP(dport=53) / \ + DNS_(rd=1, qd=DNSQR(qname=query, qtype=query_type)) + dns_query = IP(raw(dns_query)) + + response = server.build_dns_response(dns_query) + if other_query is None: + data = answer.find_rdataset(query, query_type) + else: + data = answer.find_rdataset(other_query, query_type) + + if error is None: + assert len(data) == response[DNS_].ancount + for i in range(response[DNS_].ancount): + # DEBUGGING REQUIRED FOR SCAPY UPGRADES to field types + # print("Comparison check") + # print(type(response[DNS_].an[i].type)) + # print(response[DNS_].an[i].type) + # print(response[DNS_].an[i].show()) + # print(type(response[DNS_].an[i].rdata)) + # print(response[DNS_].an[i].rdata) + + if response[DNS_].an[i].type == 16: # TXT + assert get_value(data[i], query_type) == response[DNS_].an[i].rdata[0] + continue + elif response[DNS_].an[i].type == 1: # A + assert get_value(data[i], query_type) == response[DNS_].an[i].rdata + continue + assert get_value(data[i], query_type) == response[DNS_].an[i].rdata.decode('utf-8') + elif error == RECORD_COUNT_ERROR: + assert len(data) != response[DNS_].ancount + elif error == RECORD_VALUE_ERROR: + assert len(data) == response[DNS_].ancount + for i in range(response[DNS_].ancount): + # DEBUGGING REQUIRED FOR SCAPY UPGRADES to field types + # print("Comparison check") + # print(type(response[DNS_].an[i].type)) + # print(response[DNS_].an[i].type) + # print(response[DNS_].an[i].show()) + # print(type(response[DNS_].an[i].rdata)) + # print(response[DNS_].an[i].rdata) + + if response[DNS_].an[i].type == 16: # TXT + assert get_value(data[i], query_type) != response[DNS_].an[i].rdata[0] + continue + elif response[DNS_].an[i].type == 1: # A + assert get_value(data[i], query_type) != response[DNS_].an[i].rdata + continue + assert get_value(data[i], query_type) != response[DNS_].an[i].rdata.decode('utf-8') + + if authority is True: + assert response[DNS_].aa == 1 + + + diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py new file mode 100644 index 0000000..4aa6e45 --- /dev/null +++ b/tests/test_duplicate.py @@ -0,0 +1,24 @@ +import sys +# Include the root of the project +sys.path.append("..") + +import actions.duplicate +import actions.packet +import actions.strategy +import actions.utils +import evolve + +from scapy.all import IP, TCP + + +def test_duplicate(logger): + """ + Tests the duplicate action primitive. + """ + duplicate = actions.duplicate.DuplicateAction() + assert str(duplicate) == "duplicate", "Duplicate returned incorrect string representation: %s" % str(duplicate) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) + packet1, packet2 = duplicate.run(packet, logger) + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + duplicate.mutate() diff --git a/tests/test_engine.py b/tests/test_engine.py index 88584c1..57f46dc 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,10 +1,14 @@ import os import sys +import pytest + +from scapy.all import * # Add the path to the engine so we can import it BASEPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(BASEPATH) +import actions.packet import engine def test_engine(): @@ -17,10 +21,47 @@ def test_engine(): strategy = "[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/" # Create the engine in debug mode - with engine.Engine(port, strategy, log_level="debug") as eng: + with engine.Engine(port, strategy, log_level="debug") as _: os.system("curl http://example.com?q=ultrasurf") +def test_default_args(): + """ + Tests engine can be created without specifying all of the args + """ + with engine.Engine(80, "") as eng: + assert eng.output_directory == "trials" + assert eng.environment_id is not None + + +# Test does not generate a RA, it times out - need a different way +@pytest.mark.skip() +def test_detect_rstacks(): + """ + Tests the engine detects & records seeing RA packets + """ + with engine.Engine(80, "") as eng: + os.system("curl 8.8.8.8:80") + assert eng.censorship_detected + + +def test_nat_unit(): + """ + Test NAT functionality + """ + forwarder = { + "sender_ip" : "1.1.1.1", + "routing_ip": "2.2.2.2", + "forward_ip": "3.3.3.3" + } + pkt = IP(src="1.1.1.1", dst="2.2.2.2")/TCP()/Raw("test") + packet = actions.packet.Packet(pkt) + eng = engine.Engine(80, "", forwarder=forwarder) + eng.do_nat(packet) + packet[IP].src == "2.2.2.2" + packet[IP].dst == "3.3.3.3" + eng.mysend(packet) + def test_engine_sleep(): """ Basic engine test with sleep action @@ -34,6 +75,13 @@ def test_engine_sleep(): with engine.Engine(port, strategy, log_level="info") as eng: os.system("curl http://example.com?q=ultrasurf") + +@pytest.mark.skip() +def test_engine_sleep_inbound(): + """ + Basic engine test with sleep action inbound. + """ + port = 80 # Strategy to use in opposite direction strategy = "\/ [TCP:flags:SA]-sleep{1}-|" @@ -42,7 +90,6 @@ def test_engine_sleep(): os.system("curl http://example.com?q=ultrasurf") - def test_engine_trace(): """ Basic engine test with trace diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py new file mode 100644 index 0000000..dfcaf1a --- /dev/null +++ b/tests/test_evaluator.py @@ -0,0 +1,844 @@ +import logging +import os +import pytest +import tempfile + +import actions.tree +import actions.drop +import actions.tamper +import actions.duplicate +import actions.utils +import actions.strategy +import evaluator +import evolve +import common + +import netifaces +from scapy.all import IP, TCP + +from pprint import pprint + +@pytest.mark.parametrize("extra_args", [[], ["--no-fitness-file"], ["--workers", "2"]], ids=["with_fitness_file", "no_fitness_file", "two_workers"]) +def test_evaluator_http_client(logger, extra_args): + """ + Tests http plugin client. + """ + with tempfile.TemporaryDirectory() as output_dir: + cmd = [ + "--test-type", "http", + "--port", "80", + "--external-server", + "--server", "http://google.com", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", output_dir + ] + cmd += extra_args + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 4 + assert inds[0].fitness == 389 # -10 for unused, -1 for size + assert inds[1].fitness == 400 + assert inds[2].fitness == 399 # -1 for size + assert inds[3].fitness == -480 + for ind in inds: + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".fitness")) + +@pytest.mark.parametrize("extra_args", [[], ["--use-tcp"]], ids=["udp", "tcp"]) +def test_evaluator_dns_client_external_server(logger, extra_args): + """ + Tests http plugin client. + """ + with tempfile.TemporaryDirectory() as output_dir: + cmd = [ + "--test-type", "dns", + "--external-server", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", output_dir + ] + cmd += extra_args + tester = evaluator.Evaluator(cmd, logger) + if "--use-tcp" not in cmd: + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[UDP:dport:53]-sleep{1}-|", + "[UDP:dport:53]-drop-|", # strategy that will break query + "[UDP:dport:53]-tamper{DNS:qd:compress}-|" + ] + else: + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|", # strategy that will break query + # "[TCP:flags:PA]-tamper{DNS:qd:compress}-|" # Not implemented due to TCP protocol limitations + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + + # Special case for UDP + assert len(inds) == 5 if "--use-tcp" not in cmd else 4 + + assert inds[0].fitness == 389 # -10 for unused, -1 for size + assert inds[1].fitness == 400 + assert inds[2].fitness == 399 # -1 for size + assert inds[3].fitness == -400 + + if "--use-tcp" not in cmd: + assert inds[4].fitness > 0 + + for ind in inds: + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".engine.log")) + if ind.fitness > 0: + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".dnsresult")) + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".fitness")) + + +def test_evaluator_censor_log_on_debug(logger): + """ + Tests http plugin client. + """ + print("Test testing a failing strategy and a successful strategies, dumping logs on success and failure.") + cmd = [ + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--log-on-fail", + "--log-on-success", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/", + "\/ [TCP:flags:R]-drop-|", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 2 + + finally: + print("Test shutting down any lingering containers.") + common.clean_containers() + + +def test_evaluator_censor(logger): + """ + Tests http plugin client. + """ + cmd = [ + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 4 + assert inds[0].fitness == -370 # -10 for unused + assert inds[1].fitness == -360 + assert inds[2].fitness == -360 + assert inds[3].fitness == -480 + for ind in inds: + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "flags", ind.environment_id + ".fitness")) + + finally: + print("Test shutting down any lingering containers.") + common.clean_containers() + + +def test_evaluator_censor_echo_debug(logger): + """ + Tests evaluator handling of debug mode. + """ + evaluator_censor_echo_common(logger, "debug") + + +def test_evaluator_censor_echo(logger): + """ + Tests echo plugin client. + """ + evaluator_censor_echo_common(logger, actions.utils.CONSOLE_LOG_LEVEL) + + +def evaluator_censor_echo_common(logger, log_level): + """ + Common test for test_evaluator_censor_echo and test_evaluator_censor_echo_debug + to handle both log levels. + """ + original_level = actions.utils.CONSOLE_LOG_LEVEL + logger.setLevel(log_level.upper()) + cmd = [ + "--test-type", "echo", + "--censor", "censor2", + "--log", log_level, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 4 + assert inds[0].fitness == -370 # -10 for unused + assert inds[1].fitness == -360 + assert inds[2].fitness == -360 + assert inds[3].fitness == -400 + for ind in inds: + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "flags", ind.environment_id + ".fitness")) + + finally: + logger.setLevel(original_level.upper()) + print("Test shutting down any lingering containers.") + common.clean_containers() + + +def test_evaluator_censor_discard_debug(logger): + """ + Tests evaluator's handling of debug mode + """ + test_evaluator_censor_discard(logger, "debug") + + +def test_evaluator_censor_discard(logger, log_level="info"): + """ + Tests discard plugin client for basic functionality. Discard is not + used regularly. + """ + logger.setLevel(log_level.upper()) + cmd = [ + "--test-type", "discard", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [TCP:flags:R]-drop-|", # strategy that will beat censor + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + # This is not tested, as we know it will not pass. See plugins/discard/client.py for an explanation + #"[TCP:flags:PA]-drop-| [TCP:flags:FPA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 3 + assert inds[0].fitness > 0 + assert inds[1].fitness == -410 # -10 for unused + assert inds[2].fitness == -400 + for ind in inds: + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "flags", ind.environment_id + ".fitness")) + + finally: + print("Test shutting down any lingering containers.") + os.system("docker stop server_main; docker stop censor_main; docker stop client_main") + + +def test_evaluator_censor_workers(logger): + """ + Tests http plugin client. + """ + cmd = [ + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--workers", "2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|", # strategy that will break TCP connection + "\/ [TCP:flags:R]-drop-|" + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 5 + assert inds[0].fitness == -370 # -10 for unused + assert inds[1].fitness == -360 + assert inds[2].fitness == -360 + assert inds[3].fitness == -480 + assert inds[4].fitness == 399 + for ind in inds: + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(actions.utils.RUN_DIRECTORY, "flags", ind.environment_id + ".fitness")) + + finally: + print("Test shutting down any lingering containers.") + os.system("docker stop server_0; docker stop censor_0; docker stop client_0") + os.system("docker stop server_1; docker stop censor_1; docker stop client_1") + + +def test_evaluator_http_client_skip_empty(logger): + """ + Tests http plugin client. + """ + with tempfile.TemporaryDirectory() as output_dir: + cmd = [ + "--test-type", "http", + "--port", "80", + "--external-server", + "--server", "http://google.com", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--output-directory", output_dir + ] + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-sleep{1}-|", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 4 + assert inds[0].fitness == 389 # -10 for unused, -1 for size + assert inds[1].fitness == -1000 # empty - skipped + assert inds[2].fitness == 399 # -1 for size + assert inds[3].fitness == -480 + for ind in inds: + if ind.fitness == -1000: + continue + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".fitness")) + + +def test_evaluator_http_client_injected_http(logger): + """ + Tests http plugin client. + """ + with tempfile.TemporaryDirectory() as output_dir: + cmd = [ + "--test-type", "http", + "--port", "80", + "--external-server", + "--injected-http-contains", "google", + "--server", "http://google.com", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", output_dir + ] + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 3 + assert inds[0].fitness == -370 # -10 for unused action, -360 for failing + assert inds[1].fitness == -360 + assert inds[2].fitness == -480 + for ind in inds: + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".fitness")) + + +# Many sites inside the external pool do not let multiple requests from travis, +# making the test frequently have false negative failures. +@pytest.mark.skip() +def test_evaluator_http_client_external_sites(logger): + """ + Tests http plugin client. + """ + with tempfile.TemporaryDirectory() as output_dir: + cmd = [ + "--test-type", "http", + "--external-server", + "--use-external-sites", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", output_dir + ] + tester = evaluator.Evaluator(cmd, logger) + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:PA]-drop-|" # strategy that will break TCP connection + ] + population = [actions.utils.parse(ind, logger) for ind in population] + inds = tester.evaluate(population) + assert len(inds) == 3 + assert inds[0].fitness == 389 # -10 for unused, -1 for size + assert inds[1].fitness == 400 + assert inds[2].fitness == -480 + for ind in inds: + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".client.log")) + assert os.path.exists(os.path.join(output_dir, "logs", ind.environment_id + ".engine.log")) + assert os.path.exists(os.path.join(output_dir, "flags", ind.environment_id + ".fitness")) + + +def test_evaluator_external_client_external_sites(client_worker, logger): + """ + Tests evaluator server side with external client with --use-external-sites. + """ + population = [ + "\/", + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + cmd = [ + "--test-type", "http", + "--external-server", + "--external-client", client_worker["worker"], + "--use-external-sites", + "--no-canary", + "--log-on-fail", # this test should not fail, so log if it does + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness == 400 + + +def test_evaluator_external_dns_client(client_worker, logger): + """ + Tests evaluator server side with external client with --use-external-sites. + """ + population = [ + "\/", + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + cmd = [ + "--test-type", "dns", + "--external-server", + "--external-client", client_worker["worker"], + "--log-on-fail", # this test should not fail, so log if it does + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + + +@pytest.mark.parametrize("args", [["--test-type", "http", "--port", "80"], ["--test-type", "dns", "--port", "53"]], ids=["http", "dns"]) +def test_evaluator_external_client_server_side(client_worker, logger, args): + """ + Tests evaluator server side with external client. + """ + if "http" in args or "--use-tcp" in args: + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "[TCP:flags:SA]-drop-|" # strategy that will break TCP connection + ] + else: + population = [ + "\/ [UDP:dport:100]-drop-|", # strategy with an unused action tree + "\/", + "\/ [UDP:dport:53]-drop-|" # strategy that will break query + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + cmd = [ + "--test-type", "http", + "--external-client", client_worker["worker"], + "--server-side", + "--public-ip", get_ip(), + "--timeout", "15", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + cmd += args + tester = evaluator.Evaluator(cmd, logger) + + inds = tester.evaluate(population) + assert len(inds) == 3 + assert inds[0].fitness == 389 + assert inds[1].fitness == 400 + assert inds[2].fitness < 0 + + # Request a server side without specifying the public ip - should raise an exception + cmd = [ + "--test-type", "http", + "--port", "80", + "--external-client", client_worker["worker"], + "--server-side", + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + with pytest.raises(AssertionError): + tester = evaluator.Evaluator(cmd, logger) + + +def test_evaluator_external_client(client_worker, logger): + """ + Tests evaluator server side with external client. + """ + population = [ + "\/", + ] + print(client_worker["worker"]) + population = [actions.utils.parse(ind, logger) for ind in population] + cmd = [ + "--test-type", "http", + "--port", "80", + "--external-server", + "--external-client", client_worker["worker"], + "--server", "http://google.com", + "--log-on-fail", # this test should not fail, so log when it does + "--no-canary", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness == 400 + + +def get_ip(): + """ + Helper method to get the IP address of this host. + """ + ifaces = netifaces.interfaces() + for iface in ifaces: + if "lo" in iface: + continue + info = netifaces.ifaddresses(iface) + if netifaces.AF_INET in info: + ip = info[netifaces.AF_INET][0]['addr'] + return ip + + +def test_evaluator_external_client_local_server(client_worker, logger): + """ + Tests evaluator server side with external client to a locally hosted server. + """ + population = [ + "\/", + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + cmd = [ + "--test-type", "http", + "--external-client", client_worker["worker"], + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--public-ip", get_ip(), + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness == 400 + + + +@pytest.mark.skip() +def test_evaluator_get_ip(logger): + """ + Tests evaluator skip_empty flag. + """ + # Create an evaluator + tester = evaluator.Evaluator(logger, # logger for the session + "censor2", # internal censor + None, # no external server + actions.utils.RUN_DIRECTORY, # directory to log + workers=4, # workers to use + runs=1, # only need 1 run + test_type="echo", # use echo test + skip_empty=True) # skip empty strats + ip = tester.get_ip() + tester.public_ip = "1.1.1.1" + assert tester.get_ip() == "1.1.1.1" + + +@pytest.mark.skip() +def test_evaluator_external_server(logger): + """ + Tests evaluator skip_empty flag. + """ + tester = evaluator.Evaluator(logger, # logger for the session + None, # no internal censor + "http://facebook.com", # try to talk to facebook + actions.utils.RUN_DIRECTORY, # directory to log + workers=1, # workers to use + runs=1, # only need 1 run + test_type="http", # use http test + skip_empty=False) # don't skip empty strats + + population = [ + "\/", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + + inds = tester.evaluate(population) + assert inds[0].fitness == 400 + + tester = evaluator.Evaluator(logger, # logger for the session + None, # no internal censor + "http://facebook.com", # try to talk to facebook + actions.utils.RUN_DIRECTORY, # directory to log + workers=1, # workers to use + runs=1, # only need 1 run + test_type="http", # use http test + skip_empty=True) # don't skip empty strats + + population = [ + "\/", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + + inds = tester.evaluate(population) + assert inds[0].fitness == -1000 + + tester = evaluator.Evaluator(logger, # logger for the session + None, # no internal censor + None, # no external server + actions.utils.RUN_DIRECTORY, # directory to log + use_external_sites=True, + workers=1, # workers to use + runs=1, # only need 1 run + test_type="http", # use http test + skip_empty=False) # don't skip empty strats + + population = [ + "\/", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + + inds = tester.evaluate(population) + assert inds[0].fitness == 400 + + +@pytest.mark.skip() +def test_evaluator_skip_empty(logger): + """ + Tests evaluator skip_empty flag. + """ + population = [ + "\/", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + # Create an evaluator + tester = evaluator.Evaluator(logger, # logger for the session + "censor2", # internal censor + None, # no external server + actions.utils.RUN_DIRECTORY, # directory to log + workers=4, # workers to use + runs=1, # only need 1 run + test_type="echo", # use echo test + skip_empty=True) # skip empty strats + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness == -1000 + + tester.skip_empty = False + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness == -40 + + +@pytest.mark.skip() +@pytest.mark.parametrize("test_type", ["echo", "http"]) +def test_evaluator_server_side(logger, test_type): + """ + Tests evaluator server side flag. + """ + population = [ + "\/ [TCP:flags:R]-drop-|", + ] + population = [actions.utils.parse(ind, logger) for ind in population] + # Create an evaluator with a server that only sends RSTs to the client + tester = evaluator.Evaluator(logger, # logger for the session + "censor2", # internal censor + None, # no external server + actions.utils.RUN_DIRECTORY, # directory to log + workers=4, # workers to use + runs=1, # only need 1 run for testing + test_type=test_type, # test both test types + skip_empty=True) # skip empty strats + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/ [TCP:flags:R]-drop-|" + assert inds[0].fitness > 0 + + # Switch testing to deploying the Geneva engine on the server side + tester.server_side = True + + # No RSTs are sent to the sever, so this should fail + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/ [TCP:flags:R]-drop-|" + assert inds[0].fitness < 0 + + # Switch the censor to one that sends RSTs to only the server + tester.censor = "censor5" + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/ [TCP:flags:R]-drop-|" + assert inds[0].fitness > 0 + + tester.test_type = "http" + + +@pytest.mark.skip() +@pytest.mark.parametrize("protocol", ["tcp", "udp"]) +def test_evaluator_client_dns_test(client_worker, protocol, logger): + """ + Tests DNS evaluation with external client. + """ + # Setup the population and test type + test_type = "dns_tcp" + if protocol == "udp": + test_type = "dns" + + population = [ + "\/" + ] + + population = [actions.utils.parse(ind, logger) for ind in population] + tester = evaluator.Evaluator(logger, # logger for the session + None, # no internal censor + None, # no external server + actions.utils.RUN_DIRECTORY, # directory to log + workers=1, # workers to use + runs=1, # only need 1 run for testing + external_client=False, # testing an external client + test_type=test_type, + skip_empty=False) # don't skip empty strats + + inds = tester.evaluate(population) + assert len(inds) == 1 + assert str(inds[0]).strip() == "\/" + assert inds[0].fitness > 0 + + +def test_evaluator_worker_ip_lookup(logger): + """ + Tests worker IP lookup by specifying a worker name instead of a public IP + """ + cmd = [ + "--test-type", "http", + "--public-ip", "example", + "--external-client", "example", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + test_evaluator = evaluator.Evaluator(cmd, logger) + assert test_evaluator.public_ip == "0.0.0.0" + + cmd = [ + "--test-type", "http", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + test_evaluator = evaluator.Evaluator(cmd, logger) + assert not test_evaluator.get_ip() + + +def test_evaluator_read_fitness(logger): + """ + tests evaluator read_fitness + """ + ind = actions.utils.parse("\/", logger) + ind.environment_id = "test" + cmd = [ + "--test-type", "http", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + test_evaluator = evaluator.Evaluator(cmd, logger) + test_evaluator.read_fitness(ind) + assert ind.fitness == -1000 + + +def test_evaluator_init_nat(logger): + """ + Sets up evaluator with NAT + """ + cmd = [ + "--test-type", "http", + "--sender-ip", "1.1.1.1", + "--forward-ip", "2.2.2.2", + "--routing-ip", "3.3.3.3", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + test_evaluator = evaluator.Evaluator(cmd, logger) + assert not test_evaluator.forwarder, "Evaluator set up a forwarder without --act-as-middlebox" + cmd += ["--act-as-middlebox"] + + test_evaluator = evaluator.Evaluator(cmd, logger) + + assert test_evaluator.forwarder + assert test_evaluator.forwarder["sender_ip"] == "1.1.1.1" + assert test_evaluator.forwarder["forward_ip"] == "2.2.2.2" + assert test_evaluator.forwarder["routing_ip"] == "3.3.3.3" diff --git a/tests/test_evolve.py b/tests/test_evolve.py new file mode 100644 index 0000000..adc6c75 --- /dev/null +++ b/tests/test_evolve.py @@ -0,0 +1,484 @@ +import copy +import tempfile +import traceback +import os +import pytest +import sys +# Include the root of the project +sys.path.append("..") + +import library +import common +import evolve +import evaluator +import actions.utils +import actions.packet +from actions.tamper import TamperAction +from scapy.all import IP, TCP, UDP +import random + + +def test_evolve(logger): + """ + Work in Progress + """ + strategies = [] + for strategy in library.LAB_STRATEGIES: + strategies.append(strategy["strategy"]) + logger.setLevel("ERROR") + options = {} + options["non-unique-hall"] = False + options["hall-size"] = 100000 + options["population_size"] = 500 + options["in-trees"] = 0 + options["out-trees"] = 1 + options["in-actions"] = 0 + options["out-actions"] = 3 + options["force_cleanup"] = False + options["num_generations"] = 10 + options["seed"] = None + options["elite_clones"] = 0 + options["allowed_retries"] = 20 + options["no-canary"] = True + options["load_from"] = False + options["disable_action"] = [] + hall = evolve.genetic_solve(logger, options, None) + evolve.print_results(hall, logger) + + +def test_disable_single_action(logger): + """ + Tests disabling a single action + """ + actions.packet.Packet.reset_restrictions() + try: + logger.setLevel("ERROR") + actions.action.ACTION_CACHE={} + actions.action.ACTION_CACHE["in"] = {} + actions.action.ACTION_CACHE["out"] = {} + disable_actions=["fragment", "drop", "tamper", "duplicate"] + for action in disable_actions: + print("Testing disable %s" % (str(action))) + for i in range(0, 2000): + p = evolve.generate_strategy(logger, 0, 2, 0, 4, None, disabled=[action]) + assert str(action) not in str(p) + actions.action.ACTION_CACHE={} + actions.action.ACTION_CACHE["in"] = {} + actions.action.ACTION_CACHE["out"] = {} + finally: + actions.packet.Packet.reset_restrictions() + + +def test_disable_multiple_actions(logger): + """ + Tests disabling multiple actions + """ + actions.packet.Packet.reset_restrictions() + try: + logger.setLevel("ERROR") + actions.action.ACTION_CACHE={} + actions.action.ACTION_CACHE["in"] = {} + actions.action.ACTION_CACHE["out"] = {} + disable_actions=["fragment", "drop", "tamper", "duplicate"] + for num in range(0,10): + action1 = disable_actions[random.randint(0,3)] + action2 = disable_actions[random.randint(0,3)] + action3 = disable_actions[random.randint(0,3)] + action_list = [action1, action2, action3] + print("Testing disable %s" % (str(action_list))) + for i in range(0, 1000): + p = evolve.generate_strategy(logger, 0, 2, 0, 4, None, disabled=action_list) + assert str(action1) not in str(p) + assert str(action2) not in str(p) + assert str(action3) not in str(p) + actions.action.ACTION_CACHE={} + actions.action.ACTION_CACHE["in"] = {} + actions.action.ACTION_CACHE["out"] = {} + finally: + actions.packet.Packet.reset_restrictions() + + +def assert_only(ind, field): + """ + Helper method to assert that the only tamper field in a given + individual is the given field. + """ + for forest in [ind.in_actions, ind.out_actions]: + for tree in forest: + for action in tree: + if isinstance(action, TamperAction): + assert action.field == field + + +def assert_not(ind, fields): + """ + Helper method to assert that the tamper field in a given + individual is not in the list of given fields. + """ + for forest in [ind.in_actions, ind.out_actions]: + for tree in forest: + for action in tree: + if isinstance(action, TamperAction): + assert action.field not in fields + + +@pytest.mark.parametrize("use_canary", [False, True], ids=["without_canary", "with_canary"]) +def test_disable_fields(logger, use_canary): + """ + Tests disabling fields. + """ + # Restrict evolve to using ONLY the ack field in the TCP header + try: + evolve.restrict_headers(logger, "TCP", "ack", "") + population = [] + print("Generating population pool") + canary_id = None + + # Create an evaluator + if use_canary: + cmd = [ + "--test-type", "http", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--use-external-sites", + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + canary_id = evolve.run_collection_phase(logger, tester) + assert canary_id and canary_id != -1 + + # Generate random strategies to initialize the population + for i in range(0, 2000): + p = evolve.generate_strategy(logger, 0, 2, 0, 4, None, environment_id=canary_id) + assert_only(p, "ack") + population.append(p) + + for generation in range(0, 20): + print("Starting fake generation %d" % generation) + for p in population: + p.mutate(logger) + assert_only(p, "ack") + + actions.packet.Packet.reset_restrictions() + + # Restrict evolve to using NOT the dataofs or chksum field in the TCP header + evolve.restrict_headers(logger, "TCP,UDP", "", "dataofs,chksum",) + population = [] + print("Generating population pool") + canary_id = None + # Create an evaluator + if use_canary: + cmd = [ + "--test-type", "http", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--use-external-sites", + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + canary_id = evolve.run_collection_phase(logger, tester) + assert canary_id and canary_id != -1 + + # Generate random strategies to initialize the population + for i in range(0, 2000): + p = evolve.generate_strategy(logger, 0, 2, 0, 4, None, environment_id=canary_id) + assert_not(p, ["dataofs", "chksum"]) + population.append(p) + + for generation in range(0, 20): + print("Starting fake generation %d" % generation) + for p in population: + p.mutate(logger) + assert_not(p, ["dataofs", "chksum"]) + + finally: + actions.packet.Packet.reset_restrictions() + + +@pytest.mark.parametrize("use_canary", [True, False], ids=["with_canary", "without_canary"]) +def test_population_pool(logger, use_canary): + """ + Creates a large population pool and runs them through packets. + The goal of this test is to basically fuzz the framework (and scapy) without having + to use the evaluator to do so to look for any exceptions/issues that may arise + to catch them early. + """ + logger.setLevel("ERROR") + canary_id = None + # Create an evaluator + if use_canary: + cmd = [ + "--test-type", "echo", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + canary_id = evolve.run_collection_phase(logger, tester) + + actions.packet.Packet.reset_restrictions() + population = [] + print("Generating population pool") + # Generate random strategies to initialize the population + for i in range(0, 2000): + p = evolve.generate_strategy(logger, 0, 2, 0, 4, None, environment_id=canary_id) + population.append(p) + print("Population pool generated") + packets = [ + IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S"), + IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA"), + IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="PA"), + IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA"), + IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="R"), + IP(src="127.0.0.1", dst="127.0.0.1")/UDP(sport=2222, dport=3333, chksum=0x4444), + IP(src="127.0.0.1", dst="127.0.0.1")/UDP(sport=2222, dport=3333, chksum=0x8888) + ] + packets = [actions.packet.Packet(packet) for packet in packets] + for generation in range(0, 20): + print("Starting fake generation %d" % generation) + for ind in population: + for packet in packets: + try: + ind.act_on_packet(packet, logger) + except: + traceback.print_exc() + print(str(ind)) + print(packet) + packet.show() + print(actions.packet.SUPPORTED_LAYERS) + raise + for p in population: + try: + p.mutate(logger) + except: + traceback.print_exc() + print(str(p)) + print(packet) + print(str(ind)) + raise + + +def test_eval_only(logger): + """ + Tests eval-only. + """ + cmd = [ + "--test-type", "http", + "--censor", "censor2", + "--server", "http://facebook.com", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + strat = "\/" + success_rate = evolve.eval_only(logger, strat, tester, runs=1) + assert success_rate == 0 + with tempfile.NamedTemporaryFile() as f: + f.write(str(strat).encode('utf-8')) + f.flush() + success_rate = evolve.eval_only(logger, f.name, tester, runs=1) + assert success_rate == 0 + + with tempfile.NamedTemporaryFile() as f: + # If the file is empty, we should just return None + assert not evolve.eval_only(logger, f.name, tester, runs=1) + + # Eval only with two successful strategies that are in a file + strat = "\/ [TCP:flags:R]-drop-|" + with tempfile.NamedTemporaryFile() as f: + f.write(str(strat).encode('utf-8')) + f.write(str(strat).encode('utf-8')) + f.flush() + success_rate = evolve.eval_only(logger, f.name, tester, runs=1) + assert success_rate == 1 + + +def test_mutation(logger): + """ + Tests mutation. + """ + + actions.packet.Packet.reset_restrictions() + population = [actions.utils.parse("[TCP:flags:PA]-| \/", logger)] + population[0].in_enabled = False + assert population + assert str(actions.utils.parse(str(population[0]), logger)) == str(population[0]) + # Create a hall with a strategy that has failed 5x, but not yet 10x + hall = { "[TCP:flags:PA]-drop-| \/ ": [-400] * 5 } + options = { + "cxpb": 0, + "mutpb": 1, + } + for i in range(0, 2000): + offspring = evolve.mutation_crossover(logger, population, hall, options) + assert offspring + print(offspring[0]) + if str(offspring[0]).strip() == "[TCP:flags:PA]-drop-| \/": + print("Good mutation") + break + else: + pytest.fail("Never mutated to test strategy") + print("Rejecting future mutations to [TCP:flags:PA]-drop-| \\/ ") + stred = str(actions.utils.parse("[TCP:flags:PA]-drop-| \/", logger)) + hall = { "[TCP:flags:PA]-drop-| \\/ ": [-400] * 11 } + assert stred in hall + for _ in range(0, 2000): + offspring = evolve.mutation_crossover(logger, population, hall, options) + assert offspring + assert str(offspring[0]).strip() != "[TCP:flags:PA]-drop-| \/" # should always reject this mutation + print("No rejected mutations found.") + + +def test_driver(logger): + """ + Tests evolve.py driver. + """ + cmd = [ + "--no-lock-file", + "--eval-only", "\/", + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + evolve.driver(cmd) + finally: + print("Test shutting down any lingering containers.") + common.clean_containers() + +def test_driver_lock_file(logger): + """ + Tests driver with lock file + """ + # Try with lock file + cmd = [ + "--population", "10", + "--generations", "1", + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--force-cleanup", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + try: + evolve.driver(cmd) + finally: + print("Test shutting down any lingering containers.") + common.clean_containers() + + +def test_driver_failure_cases(logger): + """ + Tests driver error handling + """ + print("Testing --no-eval with --eval-only") + # Try --no-eval and --eval-only + cmd = [ + "--population", "10", + "--generations", "1", + "--test-type", "http", + "--no-eval", + "--eval-only", "\/", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + ] + assert not evolve.driver(cmd) + + print("testing with unparseable seed") + # Try with unparseable seed + cmd = [ + "--population", "10", + "--generations", "1", + "--test-type", "http", + "--seed", "", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + ] + with pytest.raises(actions.tree.ActionTreeParseError): + evolve.driver(cmd) + + print("testing with nonexistent field") + # Try with unparseable seed + cmd = [ + "--population", "10", + "--generations", "1", + "--test-type", "http", + "--seed", "[TCP:thisdontexist:1]-drop-|", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + ] + with pytest.raises(AssertionError): + evolve.driver(cmd) + + +def test_argparse(): + """ + Normally we don't test argparsers, but the evolve.py argparser involves collecting + help strings and importing plugins, so we test it here. + """ + cmd = ["--help"] + # Should print multiple help messages and raise SystemExit + with pytest.raises(SystemExit): + evolve.get_args(cmd) + + cmd = [ + "--no-lock-file", + "--population", "10", + "--generations", "1", + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", "debug", + "--no-skip-empty", + ] + args = evolve.get_args(cmd) + assert args.no_lock_file + assert args.population == 10 + assert args.test_type == "http" + assert args.log == "debug" + + +def test_genetic_solve(): + """ + Normally we don't test argparsers, but the evolve.py argparser involves collecting + help strings and importing plugins, so we test it here. + """ + cmd = [ + "--population", "3", + "--generations", "1", + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", "info", + "--seed", "[TCP:flags:PA]-duplicate-|", + "--no-skip-empty", + ] + print(evolve.driver(cmd)) + print("testing without evaluator") + cmd = [ + "--no-eval", + "--no-lock-file", + "--population", "3", + "--generations", "1", + "--test-type", "http", + "--port", "80", + "--censor", "censor2", + "--log", "info", + "--seed", "[TCP:flags:PA]-duplicate-|", + "--no-skip-empty", + ] + print(evolve.driver(cmd)) diff --git a/tests/test_fragment.py b/tests/test_fragment.py index 3e7bcee..7c2c5de 100644 --- a/tests/test_fragment.py +++ b/tests/test_fragment.py @@ -1,5 +1,5 @@ -import logging import pytest +import logging import sys # Include the root of the project sys.path.append("..") @@ -8,13 +8,14 @@ import actions.fragment import actions.packet import actions.strategy import actions.utils +import evolve from scapy.all import IP, TCP, UDP logger = logging.getLogger("test") +MAX_UINT = 4294967295 - -def test_segment(): +def test_segment(logger): """ Tests the duplicate action primitive. """ @@ -30,8 +31,66 @@ def test_segment(): assert packet1["Raw"].load == b'da', "Left packet incorrectly fragmented" assert packet2["Raw"].load == b"ta", "Right packet incorrectly fragmented" +def test_segment_wrap(logger): + """ + Tests if segment numbers can wrap around + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + assert str(fragment) == "fragment{tcp:-1:True}", "Fragment returned incorrect string representation: %s" % str(fragment) -def test_segment_reverse(): + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP()/("data")) + packet["TCP"].seq = MAX_UINT-1 + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'da', "Left packet incorrectly fragmented" + assert packet2["Raw"].load == b"ta", "Right packet incorrectly fragmented" + assert packet1["TCP"].seq == MAX_UINT-1 + assert packet2["TCP"].seq == 0 + +def test_segment_wrap2(logger): + """ + Tests if segment numbers can wrap around testing for off-by-one + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + assert str(fragment) == "fragment{tcp:-1:True}", "Fragment returned incorrect string representation: %s" % str(fragment) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP()/("data")) + packet["TCP"].seq = MAX_UINT + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'da', "Left packet incorrectly fragmented" + assert packet2["Raw"].load == b"ta", "Right packet incorrectly fragmented" + assert packet1["TCP"].seq == MAX_UINT + assert packet2["TCP"].seq == 1 + + +def test_segment_wrap3(logger): + """ + Tests if segment numbers can wrap around testing for off-by-one + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + assert str(fragment) == "fragment{tcp:-1:True}", "Fragment returned incorrect string representation: %s" % str(fragment) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP()/("data")) + packet["TCP"].seq = MAX_UINT-2 + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'da', "Left packet incorrectly fragmented" + assert packet2["Raw"].load == b"ta", "Right packet incorrectly fragmented" + assert packet1["TCP"].seq == MAX_UINT-2 + assert packet2["TCP"].seq == MAX_UINT + + +def test_segment_reverse(logger): """ Tests the duplicate action primitive in reverse! """ @@ -48,11 +107,10 @@ def test_segment_reverse(): assert packet2["Raw"].load == b"da", "Right packet incorrectly fragmented" -def test_odd_fragment(): +def test_odd_fragment(logger): """ Tests long IP fragmentation """ - fragment = actions.fragment.FragmentAction(correct_order=True, segment=False) assert str(fragment) == "fragment{ip:-1:True}", "Fragment returned incorrect string representation: %s" % str(fragment) @@ -67,11 +125,10 @@ def test_odd_fragment(): assert packet1["Raw"].load + packet2["Raw"].load == b'\x08\xae\r\x05\x00\x00\x00d\x00\x00\x00dP\x02 \x00e\xc1\x00\x00dataisodd', "Packets fragmentation was incorrect" -def test_custom_fragment(): +def test_custom_fragment(logger): """ Tests IP fragments with custom sized lengths """ - fragment = actions.fragment.FragmentAction(correct_order=True, fragsize=3, segment=False) assert str(fragment) == "fragment{ip:3:True}", "Fragment returned incorrect string representation: %s" % str(fragment) @@ -85,11 +142,10 @@ def test_custom_fragment(): assert packet1["Raw"].load + packet2["Raw"].load == b'\x08\xae\r\x05\x00\x00\x00d\x00\x00\x00dP\x02 \x00zp\x00\x00thisissomedata', "Packets fragmentation was incorrect" -def test_reverse_fragment(): +def test_reverse_fragment(logger): """ Tests fragmentation with reversed packets """ - fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=3, segment=False) assert str(fragment) == "fragment{ip:3:False}", "Fragment returned incorrect string representation: %s" % str(fragment) @@ -103,11 +159,10 @@ def test_reverse_fragment(): assert packet2["Raw"].load + packet1["Raw"].load == b'\x08\xae\r\x05\x00\x00\x00d\x00\x00\x00dP\x02 \x00zp\x00\x00thisissomedata', "Packets fragmentation was incorrect" -def test_udp_fragment(): +def test_udp_fragment(logger): """ Tests fragmentation with reversed packets """ - fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=2, segment=False) assert str(fragment) == "fragment{ip:2:False}", "Fragment returned incorrect string representation: %s" % str(fragment) @@ -118,7 +173,21 @@ def test_udp_fragment(): assert str(packet1["Raw"].load) != str(packet2["Raw"].load), "Packets were not different" -def test_parse(): +def test_mutate(logger): + """ + Tests mutating the fragment action + """ + fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=2, segment=False) + assert str(fragment) == "fragment{ip:2:False}", "Fragment returned incorrect string representation: %s" % str(fragment) + + for _ in range(0, 200): + fragment.mutate() + fragment.parse(str(fragment), logger) + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1", proto=0x06)/TCP(sport=2222, dport=3333, chksum=0x4444)/("thisissomedata")) + packet1, packet2 = fragment.run(packet, logger) + + +def test_parse(logger): """ Tests parsing. """ @@ -168,7 +237,7 @@ def test_parse(): strat.act_on_packet(packet, logger) -def test_fallback(): +def test_fallback(logger): """ Tests fallback behavior. """ @@ -200,11 +269,10 @@ def test_fallback(): assert str(packet1) == str(packet2) -def test_ip_only_fragment(): +def test_ip_only_fragment(logger): """ Tests fragmentation without higher protocols. """ - fragment = actions.fragment.FragmentAction(correct_order=True) fragment.parse("fragment{ip:-1:True}", logger) @@ -218,3 +286,86 @@ def test_ip_only_fragment(): assert packet2["Raw"].load == b"11datadata", "Right packet incorrectly fragmented" +def test_overlapping_segment(): + """ + Basic test for overlapping segments. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:4}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11dat', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_no_overlap(): + """ + Basic test for overlapping segments with no overlap. (shouldn't ever actually happen) + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:0}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata1', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_entire_packet(): + """ + Basic test for overlapping segments overlapping entire packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:9}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segment_out_of_bounds(): + """ + Basic test for overlapping segments overlapping beyond the edge of the packet. + """ + fragment = actions.fragment.FragmentAction(correct_order=True) + fragment.parse("fragment{tcp:-1:True:20}", logger) + + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(seq=100)/("datadata11datadata")) + packet1, packet2 = fragment.run(packet, logger) + + assert id(packet1) != id(packet2), "Duplicate aliased packet objects" + + assert packet1["Raw"].load != packet2["Raw"].load, "Packets were not different" + assert packet1["Raw"].load == b'datadata11datadata', "Left packet incorrectly segmented" + assert packet2["Raw"].load == b"1datadata", "Right packet incorrectly fragmented" + + assert packet1["TCP"].seq == 100, "First packet sequence number incorrect" + assert packet2["TCP"].seq == 109, "Second packet sequence number incorrect" + +def test_overlapping_segmentation_parse(): + """ + Basic test for parsing overlapping segments. + """ + + fragment = actions.fragment.FragmentAction(correct_order=False, fragsize=2, segment=True, overlap=3) + assert str(fragment) == "fragment{tcp:2:False:3}", "Fragment returned incorrect string representation: %s" % str(fragment) diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 0000000..70c87d1 --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,37 @@ +import pytest +import sys +# Include the root of the project +sys.path.append("..") + +import library +import common +import censors.censor_driver + + +def get_tests(): + """ + Returns a list of tuples of tests of combinations of solutions and censors. + """ + tests = [] + test_type = "echo" + for solution in library.LAB_STRATEGIES: + for censor in solution["censors"]: + tests.append((solution["strategy"], censor, test_type)) + + return tests + + +@pytest.mark.parametrize("solution, censor, test_type", get_tests()) +def test_library(logger, solution, censor, test_type): + """ + Pulls each solution from the solution library and tests it against + it's corresponding censor to confirm the solution works. + """ + docker_censors = censors.censor_driver.get_censors() + if censor not in docker_censors: + pytest.skip("Censor %s is disabled." % censor) + + fitness = common.run_test(logger, solution, censor, test_type, log_on_fail=True) + # If the fitness was less than 0, the strategy failed to beat the censor + if fitness <= 0: + pytest.fail("Fitness was %d - strategy failed to beat censor." % fitness) diff --git a/tests/test_options.py b/tests/test_options.py new file mode 100644 index 0000000..ce2a568 --- /dev/null +++ b/tests/test_options.py @@ -0,0 +1,138 @@ +import copy +import sys +import pytest +# Include the root of the project +sys.path.append("..") + +import actions.strategy +import actions.utils +import actions.tamper + +from scapy.all import IP, TCP, Raw, send + + +def test_append_options(logger): + """ + Tests appending a given option + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(tamper_proto="TCP", field="options-wscale", tamper_value=50, tamper_type="replace") + lpacket, rpacket = tamper.run(packet, logger) + lpacket.show() + assert lpacket["TCP"].options == [("WScale", 50)] + + +def test_append_random_options(logger): + """ + Tests appending a given option with a random value + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-mss", tamper_type="corrupt") + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][0] == 'MSS' + assert len(lpacket["TCP"].options[0]) == 2 + +def test_tamper_options(logger): + """ + Tests tampering a given option with a given value + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-timestamp", tamper_type="replace", tamper_value=3433) + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][0] == "Timestamp" + assert lpacket["TCP"].options[0][1] == (3433, 0) + +def test_random_tamper_options(logger): + """ + Tests tampering a given option with a random value (corrupt) + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-mss", tamper_type="corrupt") + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][0] == "MSS" + if lpacket["TCP"].options[0][1] == 3453: + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][1] != 3453 + # This tests sees if it randomly chooses \xaa\xaa twice, if it did, that'd be amazing (though possible) + +def test_correct_assignment(logger): + """ + Tests that all options can be assigned + """ + for option in actions.layer.TCPLayer.scapy_options.values(): + print(option) + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-" + str(option.lower()), tamper_type="corrupt") + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][0] == option + +def test_str(logger): + """ + Tests the string representation of each + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + + tamper = actions.tamper.TamperAction(None, field="options-mss", tamper_value=39584, tamper_type="replace") + assert str(tamper) == "tamper{TCP:options-mss:replace:39584}" + +def test_parse(logger): + """ + Tests the ability to parse + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-mss") + assert tamper.parse("TCP:options-mss:corrupt", logger) + assert str(tamper) == "tamper{TCP:options-mss:corrupt}" + +def test_parse_run(logger): + """ + Tests the ability to parse + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None) + assert tamper.parse("TCP:options-mss:corrupt", logger) + + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][1] != 0 + +def test_parse_num(logger): + """ + Tests parsing integers + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, tamper_type="options") + assert tamper.parse("TCP:options-mss:replace:1440", logger) + + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][1] == 1440 + +def test_option_8(logger): + """ + Tests options 7 + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None) + assert tamper.parse("TCP:options-timestamp:replace:40000", logger) + + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][1] == (40000, 0) + +def test_option_1(logger): + """ + Tests option 1 + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, tamper_type="options") + assert tamper.parse("TCP:options-nop:corrupt", logger) + + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options[0][1] == () + +def test_md5options(logger): + """ + Tests appending a given option - the md5header + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/("data")) + tamper = actions.tamper.TamperAction(None, field="options-md5header", tamper_value=b'\xee\xee\xee\xee\xee\xee\xee\xee', tamper_type="replace") + lpacket, rpacket = tamper.run(packet, logger) + assert lpacket["TCP"].options == [(19, b'\xee\xee\xee\xee\xee\xee\xee\xee')] diff --git a/tests/test_packet.py b/tests/test_packet.py index 63ba0eb..4a81372 100644 --- a/tests/test_packet.py +++ b/tests/test_packet.py @@ -1,13 +1,11 @@ -import logging import pytest import actions.packet import actions.layer +import evolve from scapy.all import IP, TCP, UDP, DNS, DNSQR, Raw, DNSRR -logger = logging.getLogger("test") - def test_parse_layers(): """ @@ -65,7 +63,7 @@ def test_load(): Tests loads. """ tcp = actions.layer.TCPLayer(TCP()) - assert tcp.gen("load") + load = tcp.gen("load") pkt = IP()/"datadata" p = actions.packet.Packet(pkt) assert p.get("IP", "load") == "datadata" @@ -95,7 +93,7 @@ def test_load(): assert p2.get("IP", "chksum") == None -def test_parse_load(): +def test_parse_load(logger): """ Tests load parsing. """ @@ -205,7 +203,7 @@ def test_multi_opts(): """ Tests various option getting/setting. """ - pkt = IP()/TCP(options=[('MSS', 1460), ('SAckOK', b''), ('Timestamp', (4154603075, 0)), ('NOP', None), ('WScale', 7)]) + pkt = IP()/TCP(options=[('MSS', 1460), ('SAckOK', b''), ('Timestamp', (4154603075, 0)), ('NOP', None), ('WScale', 7), ('md5header', b'abcd' * 8)]) packet = actions.packet.Packet(pkt) assert packet.get("TCP", "options-sackok") == '' assert packet.get("TCP", "options-mss") == 1460 @@ -246,6 +244,16 @@ def test_options_eol(): assert any(k == "EOL" for k, v in p2["TCP"].options) +def test_compression_fallback(logger): + """ + Test that compression does not touch a packet without DNS in it packet + """ + pkt = UDP() + p = actions.packet.Packet(pkt) + p2 = actions.layer.DNSLayer.dns_decompress(p, logger) + assert p2 == p, "dns_decompress changed a non DNS packet" + + def test_options_mss(): """ Tests options-eol. @@ -418,7 +426,7 @@ def test_custom_get(): assert tcp.get("TCP", "load") == "AAAA" -def test_restrict_fields(): +def test_restrict_fields(logger): """ Tests packet field restriction. """ @@ -483,12 +491,20 @@ def test_restrict_fields(): assert layer == TCP assert field == "flags" + _, proto, field, value, _ = actions.trigger.Trigger.get_rand_trigger(None, 0) + assert proto == 'TCP' + assert field == "flags" actions.packet.Packet.reset_restrictions() actions.packet.SUPPORTED_LAYERS = [ actions.layer.IPLayer, actions.layer.TCPLayer, actions.layer.UDPLayer ] + + with pytest.raises(AssertionError): + actions.packet.Packet.restrict_fields(logger, ["TCP", "IP"], ["notathing"], ["notathing"]) + actions.packet.Packet.reset_restrictions() + actions.packet.Packet.restrict_fields(logger, ["TCP", "IP"], [], ["sport", "dport", "seq", "src"]) packet = actions.packet.Packet(pkt) packet = packet.copy() @@ -521,6 +537,10 @@ def test_restrict_fields(): assert layer in [TCP, IP] assert field not in ["sport", "dport", "seq", "src"] + _, proto, field, value, _ = actions.trigger.Trigger.get_rand_trigger(None, 0) + assert proto in ['TCP', 'IP'] + assert field not in ["sport", "dport", "seq", "src"] + actions.packet.Packet.reset_restrictions() actions.packet.SUPPORTED_LAYERS = [ actions.layer.IPLayer, @@ -528,7 +548,7 @@ def test_restrict_fields(): actions.layer.UDPLayer ] - actions.packet.Packet.restrict_fields(logger, ["IP", "UDP", "DNS"], [], ["version"]) + evolve.restrict_headers(logger, "ip,udp,dns", "", "version") packet = actions.packet.Packet(pkt) proto, field, value = packet.get_random() assert proto.__name__ in ["IP", "UDP"] diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..021483e --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,77 @@ +import pytest +import sys +# Include the root of the project +sys.path.append("..") + +import actions.strategy +import actions.utils +import library + + +EDGE_CASES = [ + "[TCP:flags:A]-| \/", + "\/ [TCP:flags:A]-|", + "[TCP:flags:A]-duplicate(duplicate(duplicate(duplicate,),),)-| \/", + "[IP:version:4]-| \/", + "[TCP:flags:A]-duplicate(tamper{TCP:flags:corrupt}(duplicate(duplicate,),),)-| \/", + "[TCP:flags:A]-tamper{TCP:flags:replace:S}(duplicate,)-| \/", + # --- Tamper value tests --- + # Tamper value should an empty string + "[IP:frag:0]-fragment{tcp:-1:False}(drop,tamper{TCP:options-altchksum:replace:})-| \/", + + # Tamper value should be "074" and be a string + "[IP:ihl:0]-fragment{tcp:-1:True}(duplicate,tamper{IP:load:replace:074})-| \/" +] + + +def get_tests(): + """ + Returns a list of tuples of tests of combinations of solutions and censors. + """ + tests = [] + for solution in library.LAB_STRATEGIES: + tests.append(solution["strategy"]) + + for strategy in EDGE_CASES: + tests.append(strategy) + + return tests + + +@pytest.mark.parametrize("solution", get_tests()) +def test_library(solution, logger): + """ + Pulls each solution from the solution library and tests it against + it's corresponding censor to confirm the solution works. + """ + # Parse the string representation of the solution + strat = actions.utils.parse(solution, logger) + logger.info("Parsed strategy %s" % (str(strat))) + + # Confirm the parsing was correct + assert str(strat).strip() == solution, "Failed to correctly parse given strategy" + +def test_quotes(logger): + """ + Tests that it properly handles strategies with leading/ending quotes. + """ + assert "\/" == str(actions.utils.parse("\"\/\"", logger)).strip() + assert "\/ [TCP:flags:A]-drop-|" == str(actions.utils.parse("\"\/ [TCP:flags:A]-drop-|\"", logger)).strip() + +def test_failures(logger): + """ + Tests that properly fails to parse strategies + """ + + with pytest.raises(actions.tree.ActionTreeParseError): + actions.utils.parse("asdfasdf", logger) + + with pytest.raises(actions.tree.ActionTreeParseError): + actions.utils.parse("[]-asdfasdf", logger) + + # Field doesn't exist + with pytest.raises(AssertionError): + actions.utils.parse("[TCP:thing:1]-nooooooope", logger) + + assert actions.utils.parse("", logger) is not None + assert " \/ " == str(actions.utils.parse("", logger)) diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..69653ad --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,52 @@ +""" +Miscellaneous tests for plugins & related functionality +""" +import copy +import sys +import requests +import os +import pytest + +# Many sites inside the external pool do not let multiple requests from travis, +# making the test frequently have false negative failures. +@pytest.mark.skip() +def test_testserver(logger): + """ + Tests the TestServer class + """ + path = os.path.join("plugins", "http") + if path not in sys.path: + sys.path.append(path) + from plugins.http.plugin import TestServer + for _ in range(10): + site = None + with TestServer(site, None, {}, logger) as site: + logger.info("Got site: %s" % site) + req = requests.get(site, timeout=10) + req.raise_for_status() + logger.info("Success") + + import plugins.http.plugin + orig = plugins.http.plugin.TEST_SITES + try: + # Overwrite the test sites with a failure case and a success case + plugins.http.plugin.TEST_SITES = ["http://nononono.no", "http://example.com"] + plugins.http.plugin.JAIL_TRACKER = {} + for site in plugins.http.plugin.TEST_SITES: + plugins.http.plugin.JAIL_TRACKER[site] = 0 + + site = None + with TestServer(site, None, {}, logger) as site: + assert site != "http://nononono.no" + logger.info("Got site: %s" % site) + req = requests.get(site, timeout=10) + req.raise_for_status() + logger.info("Success") + finally: + plugins.http.plugin.TEST_SITES = copy.deepcopy(orig) + plugins.http.plugin.JAIL_TRACKER = {} + for site in plugins.http.plugin.TEST_SITES: + plugins.http.plugin.JAIL_TRACKER[site] = 0 + + + diff --git a/tests/test_population_files.py b/tests/test_population_files.py new file mode 100644 index 0000000..ca5ab46 --- /dev/null +++ b/tests/test_population_files.py @@ -0,0 +1,166 @@ +import sys +import pytest +sys.path.append("..") # Include the root of the project +import evolve +import os +import actions.utils + +# Test Files Directory Setup +test_files_directory = os.path.join("test_files") +if not os.path.exists(test_files_directory): + os.mkdir(test_files_directory) + + +def check_one_file(logger, evolve_options, filename, population): + """ + Checks if the population in the file matches the population specified + """ + + # Number of lines in the file should match the population size + lines = sum(1 for line in open(filename)) + assert lines == len(population) + + # Loading the contents in the generation file should be equal to each random strategy + file_strategies = evolve.load_generation(logger, filename) + # Write the output of the loaded generation file for debugging purposes in case the test fails + output_file = filename + ".output" + evolve.write_generation(output_file, file_strategies) + + for index, strategy in enumerate(file_strategies): + assert str(strategy) == str(population[index]) + + # Initializing the population without the "load_from" option should not be equal in total + random_strategies = evolve.initialize_population(logger, evolve_options, None) + file_strategies_str, population_str = '', '' + + for strategy in random_strategies: + file_strategies_str += str(strategy) + + for strategy in population: + population_str += str(strategy) + + assert file_strategies_str != population_str + + # Initializing the population with the "load_from" option should be equal for each strategy + evolve_options["load_from"] = filename + file_strategies = evolve.initialize_population(logger, evolve_options, None) + for index, individual in enumerate(file_strategies): + assert str(individual) == str(population[index]) + + +def test_save_and_load_generation(logger): + """ + Generate random strategies (total number = generations * options["population_size"]), + writes the population to a file and then checks the file contents to see if it + matches the correct population when it is parsed back into the program + """ + + generations = 2 + + options = {} + options["population_size"] = 10000 + options["in-trees"] = 0 + options["out-trees"] = 1 + options["in-actions"] = 0 + options["out-actions"] = 3 + options["library"] = False + options["seed"] = None + + for generation_index in range(generations): + population = [] + population_str = '' + + # Generate random strategies to initialize the population + for i in range(options["population_size"]): + p = evolve.generate_strategy(logger, options["in-trees"], options["out-trees"], options["in-actions"], + options["out-actions"], + options["seed"], environment_id=None) + actions.utils.parse(str(p), logger) + population.append(p) + if i == options["population_size"] - 1: + population_str += str(p) + else: + population_str += str(p) + "\n" + + # Write the generation file + filename = os.path.join(test_files_directory, "generation" + str(generation_index)) + evolve.write_generation(filename, population) + + check_one_file(logger, options, filename, population) + + +def test_evolve_load_generation(logger): + """ + Generate random strategies (total number = generations * options["population_size"]), + writes the population to a file and then checks the file contents to see if it + matches the correct population when it is parsed back into the program + """ + + generations = 2 + actions.packet.Packet.reset_restrictions() + + options = {} + options["population_size"] = 2 + options["in-trees"] = 0 + options["out-trees"] = 1 + options["in-actions"] = 0 + options["out-actions"] = 3 + options["library"] = False + options["seed"] = None + + for generation_index in range(generations): + population = [] + population_str = '' + + # Generate random strategies to initialize the population + for i in range(options["population_size"]): + p = evolve.generate_strategy(logger, options["in-trees"], options["out-trees"], options["in-actions"], + options["out-actions"], + options["seed"], environment_id=None) + print(str(p)) + actions.utils.parse(str(p), logger) + population.append(p) + if i == options["population_size"] - 1: + population_str += str(p) + else: + population_str += str(p) + "\n" + + # Write the generation file + filename = os.path.join(test_files_directory, "generation" + str(generation_index)) + evolve.write_generation(filename, population) + + cmd = [ + "--population", "3", + "--generations", "1", + "--test-type", "http", + "--load-from", filename, + "--port", "80", + "--protos", "ip,udp,tcp,dns,dnsqr", + "--censor", "censor2", + "--log", "debug", + "--no-skip-empty", + ] + print(evolve.driver(cmd)) + + +@pytest.mark.skip +def test_one_file(): + """ + Used for manual testing. Loads a specified file in the + test_files_directory and checks it + """ + # Set filename here + filename = os.path.join(test_files_directory, "generation0") + + options = {} + options["population_size"] = 5 + options["in-trees"] = 0 + options["out-trees"] = 1 + options["in-actions"] = 0 + options["out-actions"] = 3 + options["library"] = False + options["seed"] = None + + population = evolve.load_generation(filename) + + check_one_file(options, filename, population) diff --git a/tests/test_sleep.py b/tests/test_sleep.py index 2b5b2c0..c3b6422 100644 --- a/tests/test_sleep.py +++ b/tests/test_sleep.py @@ -1,6 +1,5 @@ -import logging - from scapy.all import IP, TCP +import evolve import actions.utils import actions.strategy import actions.packet @@ -10,12 +9,10 @@ import sys sys.path.append("..") -def test_basic_sleep(): +def test_basic_sleep(logger): """ Tests the sleep action primitive """ - logger = logging.getLogger("test") - sleep = actions.sleep.SleepAction(.5) assert str(sleep) == "sleep{0.5}", "Sleep returned incorrect string representation: %s" % str(sleep) @@ -24,12 +21,10 @@ def test_basic_sleep(): assert packet1.sleep == .5, "Packet had wrong sleep value" -def test_sleep_str_parse(): +def test_sleep_str_parse(logger): """ Tests stringing and parsing a sleep action with a float sleep time """ - logger = logging.getLogger("test") - strat = actions.utils.parse("[TCP:flags:A]-sleep{0.5}-|", logger) assert strat.out_actions[0].action_root.time == .5 diff --git a/tests/test_strategy.py b/tests/test_strategy.py index 9e129e9..12e40a3 100644 --- a/tests/test_strategy.py +++ b/tests/test_strategy.py @@ -4,18 +4,61 @@ import pytest import actions.tree import actions.drop import actions.tamper -import actions.trace import actions.duplicate import actions.sleep import actions.utils import actions.strategy +import evaluator +import evolve -from scapy.all import IP, TCP - -logger = logging.getLogger("test") +from scapy.all import IP, TCP, Raw -def test_run(): +def test_mate(logger): + """ + Tests string representation. + """ + strat1 = actions.utils.parse("\/", logger) + strat2 = actions.utils.parse("\/", logger) + assert not actions.strategy.mate(strat1, strat2, 1) + + strat1 = actions.utils.parse("[TCP:flags:R]-duplicate-| \/", logger) + strat2 = actions.utils.parse("[TCP:flags:S]-drop-| \/", logger) + + # Mate with 100% probability + actions.strategy.mate(strat1, strat2, 1) + assert str(strat1).strip() == "[TCP:flags:R]-drop-| \/" + assert str(strat2).strip() == "[TCP:flags:S]-duplicate-| \/" + + strat1 = actions.utils.parse("[TCP:flags:R]-duplicate(drop,drop)-| \/", logger) + strat2 = actions.utils.parse("[TCP:flags:S]-drop-| \/", logger) + assert str(strat1).strip() == "[TCP:flags:R]-duplicate(drop,drop)-| \/" + assert str(strat2).strip() == "[TCP:flags:S]-drop-| \/" + + # Mate with 100% probability + actions.strategy.mate(strat1, strat2, 1) + assert str(strat1).strip() in ["[TCP:flags:R]-duplicate(drop,drop)-| \/", + "[TCP:flags:R]-drop-| \/"] + assert str(strat2).strip() in ["[TCP:flags:S]-duplicate(drop,drop)-| \/", + "[TCP:flags:S]-drop-| \/"] + + # Cannot have a strategy with a space in it - malformed + with pytest.raises(AssertionError): + actions.utils.parse("[TCP:flags:R]-duplicate(drop, drop)-| \/", logger) + + +def test_init(logger): + """ + Tests various strategy initialization. + """ + # 1 inbound tree with 1 action, zero outbound trees + strat = actions.strategy.Strategy([], []).initialize(logger, 1, 0, 1, 0, None) + s = "[TCP:flags:R]-drop-| \/" + # initialize with a seed + assert str(actions.strategy.Strategy([], []).initialize(logger, 1, 1, 1, 1, s)).strip() == s + + +def test_run(logger): """ Tests strategy execution. """ @@ -57,38 +100,110 @@ def test_run(): packets = strat4.act_on_packet(p1, logger) # Will fail with scapy 2.4.2 if packet is reparsed - strat5 = actions.utils.parse("\"[TCP:options-eol:]-tamper{TCP:load:replace:o}(tamper{TCP:dataofs:replace:11},)-| \/\"", logger) + strat5 = actions.utils.parse("[TCP:options-eol:]-tamper{TCP:load:replace:o}(tamper{TCP:dataofs:replace:11},)-| \/", logger) p1 = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) packets = strat5.act_on_packet(p1, logger) -def test_pretty_print(): +def test_mutate(): + """ + Mutates some stratiges + """ + logger = logging.getLogger("test") + logger.setLevel(logging.ERROR) + strat1 = actions.utils.parse("\/", logger) + strat1.environment_id = 1000 + strat1.mutate(logger) + assert len(strat1.out_actions) == 1 + assert len(strat1.in_actions) == 1 + assert strat1.out_actions[0].environment_id == 1000 + strat1.out_actions[0].mutate() + assert strat1.out_actions[0].environment_id == 1000 + + +def test_pretty_print(logger): """ Tests if the string representation of this strategy is correct """ - logger = logging.getLogger("test") strat = actions.utils.parse("[TCP:flags:A]-duplicate(tamper{TCP:flags:replace:R}(tamper{TCP:chksum:corrupt},),)-| \/ ", logger) correct = "TCP:flags:A\nduplicate\n├── tamper{TCP:flags:replace:R}\n│ └── tamper{TCP:chksum:corrupt}\n│ └── ===> \n└── ===> \n \n \/ \n " assert strat.pretty_print() == correct -def test_sleep_parse_handling(): +def test_collection(logger): + """ + Tests collection phase. + """ + # Create an evaluator + cmd = [ + "--test-type", "echo", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + + tester = evaluator.Evaluator(cmd, logger) + + canary = evolve.generate_strategy(logger, 0, 0, 0, 0, None) + environment_id = tester.canary_phase(canary) + packets = actions.utils.read_packets(environment_id) + assert packets + test_pop = [] + for _ in range(0, 5): + test_pop.append(evolve.generate_strategy(logger, 0, 0, 0, 0, None)) + environment_id = evolve.run_collection_phase(logger, tester) + packets = actions.utils.read_packets(environment_id) + assert packets + assert len(packets) > 1 + + +def test_sleep_parse_handling(logger): """ Tests that the sleep action handles bad parsing. """ - print("Testing incorrect parsing:") assert not actions.sleep.SleepAction().parse("THISHSOULDFAIL", logger) assert actions.sleep.SleepAction().parse("10.5", logger) -def test_trace_parse_handling(): +def test_get_from_fuzzed_or_real(logger): """ - Tests that the sleep action handles bad parsing. + Tests utils.get_from_fuzzed_or_real_packet(environment_id, real_packet_probability): """ + # Create an evaluator + cmd = [ + "--test-type", "echo", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] - print("Testing incorrect parsing:") - assert not actions.trace.TraceAction().parse("5:4", logger) - assert not actions.trace.TraceAction().parse("THISHOULDFAIL", logger) - assert not actions.trace.TraceAction().parse("", logger) + tester = evaluator.Evaluator(cmd, logger) + + canary = evolve.generate_strategy(logger, 0, 0, 0, 0, None) + environment_id = tester.canary_phase(canary) + for i in range(0, 100): + proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, 1) + assert proto + assert field + assert value is not None + proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(environment_id, 0) + assert proto + assert field + assert value is not None + + +def test_fail_cases(logger): + """ + Odd strategies that have caused failures in nightly testing. + """ + s = "[IP:proto:6]-tamper{IP:proto:replace:125}(fragment{tcp:48:True:26}(tamper{TCP:options-md5header:replace:37f0e737da65224ea03d46c713ed6fd2},),)-| \/ " + s = actions.utils.parse(s, logger) + + p = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")/Raw("aaaaaaaaaa")) + s.act_on_packet(p, logger) diff --git a/tests/test_tamper.py b/tests/test_tamper.py index e5fbeee..3303ceb 100644 --- a/tests/test_tamper.py +++ b/tests/test_tamper.py @@ -1,11 +1,12 @@ import copy -import logging import sys import pytest import random # Include the root of the project sys.path.append("..") +import evolve +import evaluator import actions.strategy import actions.packet import actions.utils @@ -15,10 +16,7 @@ import actions.layer from scapy.all import IP, TCP, UDP, DNS, DNSQR, sr1 -logger = logging.getLogger("test") - - -def test_tamper(): +def test_tamper(logger): """ Tests tampering with replace """ @@ -46,7 +44,7 @@ def test_tamper(): assert confirm_unchanged(packet, original, IP, []) -def test_tamper_ip(): +def test_tamper_ip(logger): """ Tests tampering with IP """ @@ -68,7 +66,7 @@ def test_tamper_ip(): assert confirm_unchanged(packet, original, IP, ["src"]) -def test_tamper_udp(): +def test_tamper_udp(logger): """ Tests tampering with UDP """ @@ -90,7 +88,7 @@ def test_tamper_udp(): assert confirm_unchanged(packet, original, IP, []) -def test_tamper_ip_ident(): +def test_tamper_ip_ident(logger): """ Tests tampering with IP and that the checksum is correctly changed """ @@ -129,7 +127,59 @@ def confirm_unchanged(packet, original, protocol, changed): return True -def test_parse_parameters(): +@pytest.mark.parametrize("use_canary", [False, True], ids=["without_canary", "with_canary"]) +def test_mutate(logger, use_canary): + """ + Tests the tamper 'replace' primitive. + """ + logger.setLevel("ERROR") + canary_id = None + # Create an evaluator + if use_canary: + cmd = [ + "--test-type", "echo", + "--censor", "censor2", + "--log", actions.utils.CONSOLE_LOG_LEVEL, + "--no-skip-empty", + "--bad-word", "facebook", + "--output-directory", actions.utils.RUN_DIRECTORY + ] + tester = evaluator.Evaluator(cmd, logger) + + canary_id = evolve.run_collection_phase(logger, tester) + + for _ in range(0, 25): + tamper = actions.tamper.TamperAction(None, field="flags", tamper_type="replace", tamper_value="R", tamper_proto="TCP") + + # Test mutation 200 times to ensure it remains stable + for _ in range(0, 200): + tamper._mutate(canary_id) + tamper2 = actions.tamper.TamperAction(None) + # Confirm tamper value was properly ._fix()-ed + val = tamper.tamper_value + for _ in range(0, 5): + assert tamper.tamper_value == val, "Tamper value is not stable." + # Create a test packet to ensure the field/proto choice was safe + if random.random() < 0.5: + test_packet = actions.packet.Packet(IP()/TCP()) + else: + test_packet = actions.packet.Packet(IP()/UDP()) + + # Check that tamper can run safely after mutation + try: + tamper.run(test_packet, logger) + except: + print(str(tamper)) + raise + + tamper._mutate_tamper_type() + + # Test that parsing tamper works - note we have to remove the tamper{} to make a call directly using tamper's parse. + tamper2.parse(str(tamper)[7:-1], logger) + assert str(tamper2) == str(tamper) + + +def test_parse_parameters(logger): """ Tests that tamper properly rejects malformed tamper actions """ @@ -139,7 +189,8 @@ def test_parse_parameters(): actions.tamper.TamperAction().parse("not:enough", logger) -def test_corrupt(): + +def test_corrupt(logger): """ Tests the tamper 'corrupt' primitive. """ @@ -166,7 +217,7 @@ def test_corrupt(): assert confirm_unchanged(packet, original, IP, []) -def test_add(): +def test_add(logger): """ Tests the tamper 'add' primitive. """ @@ -194,7 +245,7 @@ def test_add(): assert confirm_unchanged(packet, original, IP, []) -def test_decompress(): +def test_decompress(logger): """ Tests the tamper 'decompress' primitive. """ @@ -239,14 +290,10 @@ def test_decompress(): # Confirm tamper didn't corrupt anything else in the IP header assert confirm_unchanged(packet, original, IP, []) - packet = actions.packet.Packet(IP(dst="8.8.8.8")/TCP(dport=53)/DNS(qd=DNSQR(qname="maps.google.com"))) - original = packet.copy() - tamper.tamper(packet, logger) - assert bytes(packet) == bytes(original) -def test_corrupt_chksum(): +def test_corrupt_chksum(logger): """ Tests the tamper 'replace' primitive. """ @@ -275,7 +322,7 @@ def test_corrupt_chksum(): assert confirm_unchanged(packet, original, IP, []) -def test_corrupt_dataofs(): +def test_corrupt_dataofs(logger): """ Tests the tamper 'replace' primitive. """ @@ -301,7 +348,7 @@ def test_corrupt_dataofs(): assert confirm_unchanged(packet, original, IP, []) -def test_replace(): +def test_replace(logger): """ Tests the tamper 'replace' primitive. """ @@ -334,7 +381,17 @@ def test_replace(): assert confirm_unchanged(packet, original, IP, []) -def test_parse_flags(): +def test_init(): + """ + Tests initializing with no parameters + """ + tamper = actions.tamper.TamperAction(None) + assert tamper.field + assert tamper.tamper_proto + assert tamper.tamper_value is not None + + +def test_parse_flags(logger): """ Tests the tamper 'replace' primitive. """ @@ -350,7 +407,7 @@ def test_parse_flags(): @pytest.mark.parametrize("test_type", ["parsed", "direct"]) @pytest.mark.parametrize("value", ["EOL", "NOP", "Timestamp", "MSS", "WScale", "SAckOK", "SAck", "Timestamp", "AltChkSum", "AltChkSumOpt", "UTO"]) -def test_options(value, test_type): +def test_options(logger, value, test_type): """ Tests tampering options """ @@ -358,7 +415,6 @@ def test_options(value, test_type): tamper = actions.tamper.TamperAction(None, field="options-%s" % value.lower(), tamper_type="corrupt", tamper_value=bytes([12])) else: tamper = actions.tamper.TamperAction(None) - assert tamper.parse("TCP:options-%s:replace:" % value.lower(), logger) assert tamper.parse("TCP:options-%s:corrupt" % value.lower(), logger) packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) @@ -387,3 +443,23 @@ def test_options(value, test_type): break else: pytest.fail("Failed to find %s in options" % value) + + +def test_tamper_mutate_compress(logger): + """ + Tests that compress is handled right if its enabled + """ + backup = copy.deepcopy(actions.tamper.ACTIVATED_PRIMITIVES) + actions.tamper.ACTIVATED_PRIMITIVES = ["compress"] + try: + tamper = actions.tamper.TamperAction(None) + assert tamper.parse("TCP:flags:corrupt", logger) + tamper._mutate_tamper_type() + assert tamper.tamper_type == "compress" + assert tamper.tamper_proto_str == "DNS" + assert tamper.field == "qd" + packet = actions.packet.Packet(IP()/TCP()/DNS()/DNSQR()) + packet2 = tamper.tamper(packet, logger) + assert packet2 == packet + finally: + actions.tamper.ACTIVATED_PRIMITIVES = backup diff --git a/tests/test_trace.py b/tests/test_trace.py index 63233b9..97add36 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -1,36 +1,37 @@ -import logging import sys - +import pytest # Include the root of the project sys.path.append("..") - -import actions.strategy -import actions.packet -import actions.utils import actions.trace -import actions.layer +import actions.packet +import actions.strategy +import actions.utils +import evolve -from scapy.all import IP, TCP, UDP, DNS, DNSQR, sr1 +from scapy.all import IP, TCP -logger = logging.getLogger("test") - - -def test_trace_error_cases(): +def test_trace(logger): """ - Tests that trace handles edge cases. + Tests the trace action primitive. """ - # No IP header means the packet should just be returned - packet = actions.packet.Packet(TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) - trace = actions.trace.TraceAction(None) - p1, p2 = trace.run(packet, logger) - assert p2 is None - assert p1 == packet + trace = actions.trace.TraceAction(start_ttl=1, end_ttl=3) - # Mark the trace as having run already - it should not run again - trace.ran = True + assert str(trace) == "trace{1:3}", "Trace returned incorrect string representation: %s" % str(trace) packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) - p1, p2 = trace.run(packet, logger) - assert p1 is None - assert p2 is None + trace.run(packet, logger) + + print("Testing that trace will not run twice:") + assert trace.run(packet, logger) == (None, None) + + trace = actions.trace.TraceAction(start_ttl=1, end_ttl=3) + packet = actions.packet.Packet(TCP()) + assert trace.run(packet, logger) == (packet, None) + + s = "[TCP:flags:PA]-trace{1:3}-| \/ " + assert str(actions.utils.parse(s, logger)) == s + + assert not trace.parse("10:4", logger) + assert not trace.parse("10:hi", logger) + assert not trace.parse("", logger) diff --git a/tests/test_tree.py b/tests/test_tree.py index 4c8e74b..d1ae0d9 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -126,7 +126,7 @@ def test_pretty_print_send(): assert a.pretty_print() == correct_string -def test_pretty_print(): +def test_pretty_print(logger): """ Print complex tree, although difficult to test """ @@ -161,7 +161,7 @@ def test_pretty_print(): assert a.pretty_print(visual=True) assert os.path.exists("tree.png") os.remove("tree.png") - a.parse("[TCP:flags:0]-|", logging.getLogger("test")) + a.parse("[TCP:flags:0]-|", logger) a.pretty_print(visual=True) # Empty action tree assert not os.path.exists("tree.png") @@ -261,13 +261,11 @@ def test_tree(): tamper = actions.tamper.TamperAction() tamper2 = actions.tamper.TamperAction() duplicate = actions.duplicate.DuplicateAction() - assert a.get_parent(None) == (None, None) a.add_action(None) a.add_action(tamper) assert a.get_slots() == 1 a.add_action(tamper2) - assert a.get_parent(tamper2) == (tamper, "left") assert a.get_slots() == 1 a.add_action(duplicate) assert a.get_slots() == 2 @@ -276,7 +274,6 @@ def test_tree(): a = actions.tree.ActionTree("out", trigger=t) drop = actions.drop.DropAction() a.add_action(drop) - assert a.get_parent(drop) == (None, None) assert a.get_slots() == 0 add_success = a.add_action(tamper) assert not add_success @@ -294,6 +291,8 @@ def test_tree(): print(str(a)) assert len(a) == 2 assert a.get_slots() == 2 + for _ in range(100): + assert str(a.get_rand_action("out", request="DropAction")) == "drop" def test_remove(): @@ -326,7 +325,6 @@ def test_remove(): duplicate.left = tamper2 duplicate.right = tamper3 a.add_action(duplicate) - assert a.get_parent(tamper3) == (duplicate, "right") assert len(a) == 4 assert a.remove_action(duplicate) assert duplicate not in a @@ -457,6 +455,7 @@ def test_run(): duplicate.left = tamper duplicate.right = tamper2 packet = actions.packet.Packet(IP()/TCP(flags="RA")) + print("ABUT TO RUN") packets = a.run(packet, logging.getLogger("test")) assert len(packets) == 2 assert None not in packets @@ -524,6 +523,142 @@ def test_index(): assert a[-1] == tamper3 assert not a[-11] +def test_mate(): + """ + Tests mate primitive + """ + logger = logging.getLogger("test") + t = actions.trigger.Trigger("field", "flags", "TCP") + a = actions.tree.ActionTree("out", trigger=t) + assert not a.choose_one() + tamper = actions.tamper.TamperAction(field="flags", tamper_type="replace", tamper_value="S") + tamper2 = actions.tamper.TamperAction(field="flags", tamper_type="replace", tamper_value="R") + duplicate = actions.duplicate.DuplicateAction() + duplicate2 = actions.duplicate.DuplicateAction() + drop = actions.drop.DropAction() + other_a = actions.tree.ActionTree("out", trigger=t) + assert not a.mate(other_a), "Can't mate empty trees" + assert a.add_action(tamper) + assert other_a.add_action(tamper2) + assert a.choose_one() == tamper + assert other_a.choose_one() == tamper2 + assert a.get_parent(tamper) == (None, None) + assert other_a.get_parent(tamper2) == (None, None) + assert a.add_action(duplicate) + assert a.get_parent(duplicate) == (tamper, "left") + duplicate.right = drop + assert a.get_parent(drop) == (duplicate, "right") + assert other_a.add_action(duplicate2) + # Test mating a full tree with a full tree + assert str(a) == "[TCP:flags:0]-tamper{TCP:flags:replace:S}(duplicate(,drop),)-|" + assert str(other_a) == "[TCP:flags:0]-tamper{TCP:flags:replace:R}(duplicate,)-|" + assert a.swap(duplicate, other_a, duplicate2) + assert str(a).strip() == "[TCP:flags:0]-tamper{TCP:flags:replace:S}(duplicate,)-|" + assert str(other_a).strip() == "[TCP:flags:0]-tamper{TCP:flags:replace:R}(duplicate(,drop),)-|" + assert len(a) == 2 + assert len(other_a) == 3 + assert duplicate2 not in other_a + assert duplicate not in a + assert tamper.left == duplicate2 + assert tamper2.left == duplicate + assert other_a.get_parent(duplicate) == (tamper2, "left") + assert a.get_parent(duplicate2) == (tamper, "left") + assert other_a.get_parent(drop) == (duplicate, "right") + assert a.get_parent(None) == (None, None) + + # Test mating two trees with just root nodes + t = actions.trigger.Trigger("field", "flags", "TCP") + a = actions.tree.ActionTree("out", trigger=t) + assert not a.choose_one() + tamper = actions.tamper.TamperAction(field="flags", tamper_type="replace", tamper_value="S") + tamper2 = actions.tamper.TamperAction(field="flags", tamper_type="replace", tamper_value="R") + duplicate = actions.duplicate.DuplicateAction() + duplicate2 = actions.duplicate.DuplicateAction() + drop = actions.drop.DropAction() + other_a = actions.tree.ActionTree("out", trigger=t) + assert not a.mate(other_a) + assert a.add_action(duplicate) + assert other_a.add_action(duplicate2) + assert a.mate(other_a) + assert a.action_root == duplicate2 + assert other_a.action_root == duplicate + assert not duplicate.left and not duplicate.right + assert not duplicate2.left and not duplicate2.right + # Confirm that no nodes have been aliased or connected between the trees + for node in a: + for other_node in other_a: + assert not node.left == other_node + assert not node.right == other_node + + # Test mating two trees where one is empty + assert a.remove_action(duplicate2) + # This should swap the duplicate action to be the action root of the other tree + assert str(a) == "[TCP:flags:0]-|" + assert str(other_a) == "[TCP:flags:0]-duplicate-|" + assert a.mate(other_a) + assert not other_a.action_root + assert a.action_root == duplicate + assert len(a) == 1 + assert len(other_a) == 0 + # Confirm that no nodes have been aliased or connected between the trees + for node in a: + for other_node in other_a: + if other_node: + assert not node.left == other_node + assert not node.right == other_node + + assert a.parse("[TCP:flags:0]-tamper{TCP:flags:replace:S}(duplicate(,drop),)-|", logger) + drop = a.action_root.left.right + assert str(drop) == "drop" + # Note that this will return a valid ActionTree, but because it is empty, + # it is technically a False-y value, as it's length is 0 + assert other_a.parse("[TCP:flags:0]-|", logger) == other_a + + a.swap(drop, other_a, None) + assert other_a.action_root == drop + assert not a.action_root.left.right + assert str(other_a) == "[TCP:flags:0]-drop-|" + assert str(a) == "[TCP:flags:0]-tamper{TCP:flags:replace:S}(duplicate,)-|" + other_a.swap(drop, a, a.action_root.left) + # Confirm that no nodes have been aliased or connected between the trees + for node in a: + for other_node in other_a: + if other_node: + assert not node.left == other_node + assert not node.right == other_node + + assert str(other_a) == "[TCP:flags:0]-duplicate-|" + assert str(a) == "[TCP:flags:0]-tamper{TCP:flags:replace:S}(drop,)-|" + + a.parse("[TCP:flags:0]-drop-|", logger) + other_a.parse("[TCP:flags:0]-duplicate(drop,drop)-|", logger) + a_drop = a.action_root + other_duplicate = other_a.action_root + a.swap(a_drop, other_a, other_duplicate) + print(str(a)) + print(str(other_a)) + assert str(other_a) == "[TCP:flags:0]-drop-|" + assert str(a) == "[TCP:flags:0]-duplicate(drop,drop)-|" + duplicate = actions.duplicate.DuplicateAction() + duplicate2 = actions.duplicate.DuplicateAction() + drop = actions.drop.DropAction() + drop2 = actions.drop.DropAction() + drop3 = actions.drop.DropAction() + a = actions.tree.ActionTree("out", trigger=t) + a.add_action(duplicate) + a.add_action(drop) + a.add_action(drop2) + assert str(a) == "[TCP:flags:0]-duplicate(drop,drop)-|" + assert a.get_slots() == 0 + other_a = actions.tree.ActionTree("out", trigger=t) + other_a.add_action(drop3) + a.swap(drop, other_a, drop3) + assert str(a) == "[TCP:flags:0]-duplicate(drop,drop)-|" + a.swap(drop3, other_a, drop) + assert str(a) == "[TCP:flags:0]-duplicate(drop,drop)-|" + + assert a.mate(other_a) + def test_choose_one(): """ diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 062c2ec..48e6a9b 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,4 +1,3 @@ -import logging import sys # Include the root of the project sys.path.append("..") @@ -7,17 +6,35 @@ import actions.packet import actions.strategy import actions.tamper import actions.utils +import evolve from scapy.all import IP, TCP -logger = logging.getLogger("test") + +def test_mutate(): + """ + Tests the tamper 'replace' primitive. + """ + trigger = actions.trigger.Trigger("field", "flags", "TCP") + trigger.mutate(None) -def test_trigger_gas(): +def test_init(logger): + """ + Tests initialization. + """ + packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="S")) + trigger = actions.trigger.Trigger(None, None, None) + trigger.is_applicable(packet, logger) + + actions.trigger.FIXED_TRIGGER = actions.trigger.Trigger.parse("TCP:flags:SA") + assert actions.trigger.Trigger.get_rand_trigger("test", 1) == ("field", "TCP", "flags", "SA", None) + + +def test_trigger_gas(logger): """ Tests triggers having gas, including changing that gas while in use """ - packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA")) trigger = actions.trigger.Trigger("field", "flags", "TCP", trigger_value="SA", gas=1) print(trigger) @@ -45,11 +62,10 @@ def test_trigger_gas(): assert not trigger.is_applicable(packet, logger) -def test_bomb_trigger_gas(): +def test_bomb_trigger_gas(logger): """ Tests triggers having bomb gas, including changing that gas while in use """ - packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA")) trigger = actions.trigger.Trigger("field", "flags", "TCP", trigger_value="SA", gas=-1) print(trigger) @@ -78,11 +94,10 @@ def test_bomb_trigger_gas(): assert trigger.is_applicable(packet, logger) -def test_trigger_parse_gas(): +def test_trigger_parse_gas(logger): """ Tests triggers having gas, including changing that gas while in use """ - packet = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA")) @@ -113,8 +128,7 @@ def test_trigger_parse_gas(): trigger = actions.trigger.Trigger.parse("[TCP:flags:SA]") assert trigger.is_applicable(packet, logger) - -def test_bomb_trigger_parse_gas(): +def test_bomb_trigger_parse_gas(logger): """ Tests bomb triggers having gas, including changing that gas while in use """ @@ -149,3 +163,17 @@ def test_bomb_trigger_parse_gas(): # Test that it can handle leading/trailing [] trigger = actions.trigger.Trigger.parse("[TCP:flags:SA]") assert trigger.is_applicable(packet, logger) + +def test_wildcard(logger): + """ + Test wildcard trigger value + """ + packet_1 = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="A")) + packet_2 = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="SA")) + packet_3 = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="RA")) + packet_4 = actions.packet.Packet(IP(src="127.0.0.1", dst="127.0.0.1")/TCP(sport=2222, dport=3333, seq=100, ack=100, flags="P")) + trigger = actions.trigger.Trigger("field", "flags", "TCP", trigger_value="A*", gas=None) + assert trigger.is_applicable(packet_1, logger) + assert trigger.is_applicable(packet_2, logger) + assert trigger.is_applicable(packet_3, logger) + assert not trigger.is_applicable(packet_4, logger) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6f84891..903a208 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,21 +8,17 @@ import actions.strategy import actions.utils import actions.duplicate -import logging - -logger = logging.getLogger("test") - def get_test_configs(): """ Sets up the tests """ tests = [ - ("both", True, ['DuplicateAction', 'DropAction', 'SleepAction', 'TraceAction', 'TamperAction', 'FragmentAction']), - ("in", True, ['DropAction', 'TamperAction', 'SleepAction']), - ("out", True, ['DropAction', 'TamperAction', 'TraceAction', 'SleepAction', 'DuplicateAction', 'FragmentAction']), + ("both", True, ['DuplicateAction', 'TraceAction', 'DropAction', 'SleepAction', 'TamperAction', 'FragmentAction']), + ("in", True, ['DropAction', 'TamperAction']), + ("out", True, ['DropAction', 'TamperAction', 'SleepAction', 'TraceAction', 'DuplicateAction', 'FragmentAction']), ("both", False, ['DuplicateAction', 'SleepAction', 'TamperAction', 'FragmentAction']), - ("in", False, ['TamperAction', 'SleepAction']), + ("in", False, ['TamperAction']), ("out", False, ['TamperAction', 'SleepAction', 'DuplicateAction', 'FragmentAction']), ] # To ensure caching is not breaking anything, double the tests @@ -40,3 +36,35 @@ def test_get_actions(direction, allow_terminal, supported_actions): names.append(name) assert set(names) == set(supported_actions) assert len(names) == len(supported_actions) + + +def test_punish_no_engine(logger): + """ + Tests that punish_fitness properly handles no engine + """ + assert 100 == actions.utils.punish_fitness(100, logger, None) + assert 100 == actions.utils.punish_complexity(100, logger, None) + assert 100 == actions.utils.punish_unused(100, logger, None) + + +def test_write_fitness_error(logger): + """ + Tests handling of write fitness error cases + """ + with pytest.raises(ValueError): + actions.utils.write_fitness("", None, None) + + +def test_skipstrat(logger): + """ + Tests we can create and raise a SkipStrategyException + """ + with pytest.raises(actions.utils.SkipStrategyException): + raise actions.utils.SkipStrategyException("Skip this!", 100) + + +def test_import_plugin(logger): + """ + Tries to import some plugins + """ + assert actions.utils.import_plugin("http", "client") diff --git a/workers/example/example.pem b/workers/example/example.pem new file mode 100644 index 0000000..d138899 --- /dev/null +++ b/workers/example/example.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA8CThIJh9JerUfiNTETX+6psNAEWe6qSeReBE3/h7/FhjdU7s +Pwah6wp9PutTI+Gxgqrd3Xpt02rOUIv2JPySMukiiHlI4Lgn9P2OJkI/O9HkBJnB +K+CMw3U+9g4onMVnu5ZXEEml9t4nCnlm5xpLckKD16miNsv2FtFUZqihUOlU56TC +QrL+xC5gVVojS7NpGKVjYC1fFZzJ+Cz48CbZijWFZm2rSuaU8Qr24Ll4oNwxzH8Z +d2HME/Z9zcUw472mT+/GykiCHoLznV2JgpaUYlwzu/eiiNszNw/ltFR1PkmW5BYU +gtyGvJ3akde6bT+N7pb3tFGWjqBXwHRkv/02FwIDAQABAoIBAQCgRy0O0Vft3771 +dWQyv4QlV1klDlKTr02Hu7icd79g3Cl9mTBqnGWjdzvCq9Owxijw2lP8R4NXsRiE +f/XIF98IJbwLtE9k1OSuSjD+7gZAScVTIL/iJhwtNGrSVP7cK5EhjaUMVOt1Si5q +ej6pDk6+sxBF940x342fCxW28uPAGMCpudF6JqXc4ux6vQJDXz+a6RSwcl3eFt4o +3Vd/BjXaVUO+ZXJFka2XbS8AS6FpUZDxaqgV+O3RERIFVaJbtfBD0xb9ju89cOjH +a7c/nk1Sodx561A4Xo8VWIWAdlKniuAHWHncm29H45tkq+Eu0AjdBhRDuGxaaXPU +E9ZkxX9xAoGBAP1pQ88JlVA0zFxJLF2EwmrAeC4kPdYp87Y6kCrs4ZsZ2Gxu85A8 +drsEMzH/fRTVLOANQoS9UoAph0Fi/492RFMB0Od4SwLs1bpN8bAQEj8TZfJw2FRg +7GnGxcEhsPQ76pyOZAYxYCCJTQzr/Ax3f5PzD9pJkAwI9cUqn8MtBWL/AoGBAPKY +6ua72HB15VpDwiYD/LjracF6SJAxVO0vYn+q08SW/Af6fu4hqcGbW6NY6CYUdmOk +++yKt7yOOJnmGEcDnxkznKiAL3xA+vyL2MlQxfWKzdSNhYnL0cjQHflT6sPRXqKU +Ptn7y7n5H6I5vyeXVv5uIOchET3oAlxViRglFeTpAoGBAK+HQXvgRHeZCtEFsmBY +vB4bkWNMoTR+PJzF9eR1zlOT1HBPbhBDgW5LG3cPZEGQ45OSO7gcN1Ak89Ybz688 +mVjx0hTzIuo0yFpS8Q0mJK1iL2R+O6o+tKMUFxiO0LZr5jdo/em4O8Btulwv/ZdZ +EpqVay6lc8ySTEK1IpKb1bZbAoGBAOLKiBcKFu4TmUUIQTEu2wy4r1Y7jiiWpbFS +cf+t4GlRSO3ghNbzy3H+Xe3YCLbIvGCs9QKOL/Rq9IkTLTD+NrblpBd2Nm4vhTOh +PRn+lAHXMFL7tpI6Y7UaslrK05caXSEZ+6PnG+6Q8bzDxWsqtR4IMGRCao74HXTa +Vd6nlM85AoGBALiTwChcht86N2ndBNCO36QoK4aCey65r1zxbO/wcFCa07SHkrRR +Srj2kWhyGtseEA4xkhry8K+kH4WMfdqoZGtePRNAuMDp7jPVZ1Xzn7eRhAq24U+R +0PM7vn6RBOrjZMSWpDPAgHsl7kui3Mm/UTRHccmHdSVy69oCKwNR2JpL +-----END RSA PRIVATE KEY----- diff --git a/workers/example/example.pub b/workers/example/example.pub new file mode 100644 index 0000000..265ec3d --- /dev/null +++ b/workers/example/example.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwJOEgmH0l6tR+I1MRNf7qmw0ARZ7qpJ5F4ETf+Hv8WGN1Tuw/BqHrCn0+61Mj4bGCqt3dem3Tas5Qi/Yk/JIy6SKIeUjguCf0/Y4mQj870eQEmcEr4IzDdT72DiicxWe7llcQSaX23icKeWbnGktyQoPXqaI2y/YW0VRmqKFQ6VTnpMJCsv7ELmBVWiNLs2kYpWNgLV8VnMn4LPjwJtmKNYVmbatK5pTxCvbguXig3DHMfxl3YcwT9n3NxTDjvaZP78bKSIIegvOdXYmClpRiXDO796KI2zM3D+W0VHU+SZbkFhSC3Ia8ndqR17ptP43ulve0UZaOoFfAdGS//TYX geneva@geneva.com diff --git a/workers/example/worker.json b/workers/example/worker.json new file mode 100644 index 0000000..75af717 --- /dev/null +++ b/workers/example/worker.json @@ -0,0 +1,16 @@ +{ + "name": "example", + "ip": "0.0.0.0", + "hostname": "example.com", + "username": "user", + "password": null, + "port": 22, + "python": "python3", + "city": "mycensoredcity", + "keyfile": "example.pem", + "country": "mycensoredcountry", + "geneva_path": "evolving-evasion", + "censored": true, + "active": true, + "provider": null +}