diff --git a/tests/compile-test.rs b/tests/compile-test.rs
index b965dceb774..ff594f7a464 100644
--- a/tests/compile-test.rs
+++ b/tests/compile-test.rs
@@ -3,6 +3,9 @@
 extern crate compiletest_rs as compiletest;
 extern crate test;
 
+use std::ffi::OsStr;
+use std::fs;
+use std::error::Error;
 use std::env::{set_var, var};
 use std::path::{Path, PathBuf};
 
@@ -30,7 +33,7 @@ fn rustc_lib_path() -> PathBuf {
     option_env!("RUSTC_LIB_PATH").unwrap().into()
 }
 
-fn config(dir: &'static str, mode: &'static str) -> compiletest::Config {
+fn config(mode: &str, dir: &str) -> compiletest::Config {
     let mut config = compiletest::Config::default();
 
     let cfg_mode = mode.parse().expect("Invalid mode");
@@ -61,8 +64,38 @@ fn config(dir: &'static str, mode: &'static str) -> compiletest::Config {
     config
 }
 
-fn run_mode(dir: &'static str, mode: &'static str) {
-    compiletest::run_tests(&config(dir, mode));
+fn run_mode(mode: &str, dir: &str) {
+    compiletest::run_tests(&config(mode, dir));
+}
+
+fn run_ui_toml() -> Result<(), Box<Error>> {
+    let base = PathBuf::from("tests/ui-toml/").canonicalize()?;
+    for dir in fs::read_dir(&base)? {
+        let dir = dir?;
+        if !dir.file_type()?.is_dir() {
+            continue;
+        }
+        let dir_path = dir.path();
+        set_var("CARGO_MANIFEST_DIR", &dir_path);
+        let config = config("ui", "ui-toml");
+        for file in fs::read_dir(&dir_path)? {
+            let file = file?;
+            let file_path = file.path();
+            if !file.file_type()?.is_file() {
+                continue;
+            }
+            if file_path.extension() != Some(OsStr::new("rs")) {
+                continue;
+            }
+            let paths = compiletest::common::TestPaths {
+                file: file_path,
+                base: base.clone(),
+                relative_dir: dir_path.file_name().unwrap().into(),
+            };
+            compiletest::runtest::run(config.clone(), &paths);
+        }
+    }
+    Ok(())
 }
 
 fn prepare_env() {
@@ -76,4 +109,5 @@ fn compile_test() {
     prepare_env();
     run_mode("run-pass", "run-pass");
     run_mode("ui", "ui");
+    run_ui_toml().unwrap();
 }
diff --git a/tests/ui/bad_toml/clippy.toml b/tests/ui-toml/bad_toml/clippy.toml
similarity index 100%
rename from tests/ui/bad_toml/clippy.toml
rename to tests/ui-toml/bad_toml/clippy.toml
diff --git a/tests/ui/bad_toml/conf_bad_toml.rs b/tests/ui-toml/bad_toml/conf_bad_toml.rs
similarity index 100%
rename from tests/ui/bad_toml/conf_bad_toml.rs
rename to tests/ui-toml/bad_toml/conf_bad_toml.rs
diff --git a/tests/ui-toml/bad_toml/conf_bad_toml.stderr b/tests/ui-toml/bad_toml/conf_bad_toml.stderr
new file mode 100644
index 00000000000..85b3ef8612e
--- /dev/null
+++ b/tests/ui-toml/bad_toml/conf_bad_toml.stderr
@@ -0,0 +1,4 @@
+error: error reading Clippy's configuration file `$DIR/clippy.toml`: expected an equals, found an identifier at line 1
+
+error: aborting due to previous error
+
diff --git a/tests/ui/bad_toml_type/clippy.toml b/tests/ui-toml/bad_toml_type/clippy.toml
similarity index 100%
rename from tests/ui/bad_toml_type/clippy.toml
rename to tests/ui-toml/bad_toml_type/clippy.toml
diff --git a/tests/ui/bad_toml_type/conf_bad_type.rs b/tests/ui-toml/bad_toml_type/conf_bad_type.rs
similarity index 100%
rename from tests/ui/bad_toml_type/conf_bad_type.rs
rename to tests/ui-toml/bad_toml_type/conf_bad_type.rs
diff --git a/tests/ui-toml/bad_toml_type/conf_bad_type.stderr b/tests/ui-toml/bad_toml_type/conf_bad_type.stderr
new file mode 100644
index 00000000000..efd02bcbb6e
--- /dev/null
+++ b/tests/ui-toml/bad_toml_type/conf_bad_type.stderr
@@ -0,0 +1,4 @@
+error: error reading Clippy's configuration file `$DIR/clippy.toml`: invalid type: integer `42`, expected a sequence
+
+error: aborting due to previous error
+
diff --git a/tests/ui/toml_blacklist/clippy.toml b/tests/ui-toml/toml_blacklist/clippy.toml
similarity index 100%
rename from tests/ui/toml_blacklist/clippy.toml
rename to tests/ui-toml/toml_blacklist/clippy.toml
diff --git a/tests/ui/toml_blacklist/conf_french_blacklisted_name.rs b/tests/ui-toml/toml_blacklist/conf_french_blacklisted_name.rs
similarity index 100%
rename from tests/ui/toml_blacklist/conf_french_blacklisted_name.rs
rename to tests/ui-toml/toml_blacklist/conf_french_blacklisted_name.rs
diff --git a/tests/ui-toml/toml_blacklist/conf_french_blacklisted_name.stderr b/tests/ui-toml/toml_blacklist/conf_french_blacklisted_name.stderr
new file mode 100644
index 00000000000..b2b0f26b140
--- /dev/null
+++ b/tests/ui-toml/toml_blacklist/conf_french_blacklisted_name.stderr
@@ -0,0 +1,46 @@
+error: use of a blacklisted/placeholder name `toto`
+ --> $DIR/conf_french_blacklisted_name.rs:9:9
+  |
+9 | fn test(toto: ()) {}
+  |         ^^^^
+  |
+  = note: `-D blacklisted-name` implied by `-D warnings`
+
+error: use of a blacklisted/placeholder name `toto`
+  --> $DIR/conf_french_blacklisted_name.rs:12:9
+   |
+12 |     let toto = 42;
+   |         ^^^^
+
+error: use of a blacklisted/placeholder name `tata`
+  --> $DIR/conf_french_blacklisted_name.rs:13:9
+   |
+13 |     let tata = 42;
+   |         ^^^^
+
+error: use of a blacklisted/placeholder name `titi`
+  --> $DIR/conf_french_blacklisted_name.rs:14:9
+   |
+14 |     let titi = 42;
+   |         ^^^^
+
+error: use of a blacklisted/placeholder name `toto`
+  --> $DIR/conf_french_blacklisted_name.rs:20:10
+   |
+20 |         (toto, Some(tata), titi @ Some(_)) => (),
+   |          ^^^^
+
+error: use of a blacklisted/placeholder name `tata`
+  --> $DIR/conf_french_blacklisted_name.rs:20:21
+   |
+20 |         (toto, Some(tata), titi @ Some(_)) => (),
+   |                     ^^^^
+
+error: use of a blacklisted/placeholder name `titi`
+  --> $DIR/conf_french_blacklisted_name.rs:20:28
+   |
+20 |         (toto, Some(tata), titi @ Some(_)) => (),
+   |                            ^^^^
+
+error: aborting due to 7 previous errors
+
diff --git a/tests/ui/toml_unknown_key/clippy.toml b/tests/ui-toml/toml_unknown_key/clippy.toml
similarity index 100%
rename from tests/ui/toml_unknown_key/clippy.toml
rename to tests/ui-toml/toml_unknown_key/clippy.toml
diff --git a/tests/ui/toml_unknown_key/conf_unknown_key.rs b/tests/ui-toml/toml_unknown_key/conf_unknown_key.rs
similarity index 100%
rename from tests/ui/toml_unknown_key/conf_unknown_key.rs
rename to tests/ui-toml/toml_unknown_key/conf_unknown_key.rs
diff --git a/tests/ui-toml/toml_unknown_key/conf_unknown_key.stderr b/tests/ui-toml/toml_unknown_key/conf_unknown_key.stderr
new file mode 100644
index 00000000000..61e03774e32
--- /dev/null
+++ b/tests/ui-toml/toml_unknown_key/conf_unknown_key.stderr
@@ -0,0 +1,4 @@
+error: error reading Clippy's configuration file `$DIR/clippy.toml`: unknown field `foobar`, expected one of `blacklisted-names`, `cyclomatic-complexity-threshold`, `doc-valid-idents`, `too-many-arguments-threshold`, `type-complexity-threshold`, `single-char-binding-names-threshold`, `too-large-for-stack`, `enum-variant-name-threshold`, `enum-variant-size-threshold`, `verbose-bit-mask-threshold`, `literal-representation-threshold`, `third-party`
+
+error: aborting due to previous error
+
diff --git a/tests/ui-toml/update-all-references.sh b/tests/ui-toml/update-all-references.sh
new file mode 100755
index 00000000000..acc38f15fbd
--- /dev/null
+++ b/tests/ui-toml/update-all-references.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# A script to update the references for all tests. The idea is that
+# you do a run, which will generate files in the build directory
+# containing the (normalized) actual output of the compiler. You then
+# run this script, which will copy those files over. If you find
+# yourself manually editing a foo.stderr file, you're doing it wrong.
+#
+# See all `update-references.sh`, if you just want to update a single test.
+
+if [[ "$1" == "--help" || "$1" == "-h" ]]; then
+    echo "usage: $0"
+fi
+
+BUILD_DIR=$PWD/target/debug/test_build_base
+MY_DIR=$(dirname $0)
+cd $MY_DIR
+find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
diff --git a/tests/ui-toml/update-references.sh b/tests/ui-toml/update-references.sh
new file mode 100755
index 00000000000..aa99d35f7aa
--- /dev/null
+++ b/tests/ui-toml/update-references.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# A script to update the references for particular tests. The idea is
+# that you do a run, which will generate files in the build directory
+# containing the (normalized) actual output of the compiler. This
+# script will then copy that output and replace the "expected output"
+# files. You can then commit the changes.
+#
+# If you find yourself manually editing a foo.stderr file, you're
+# doing it wrong.
+
+if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
+    echo "usage: $0 <build-directory> <relative-path-to-rs-files>"
+    echo ""
+    echo "For example:"
+    echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
+fi
+
+MYDIR=$(dirname $0)
+
+BUILD_DIR="$1"
+shift
+
+while [[ "$1" != "" ]]; do
+    STDERR_NAME="${1/%.rs/.stderr}"
+    STDOUT_NAME="${1/%.rs/.stdout}"
+    shift
+    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
+           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
+        echo updating $MYDIR/$STDOUT_NAME
+        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    fi
+    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
+           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
+        echo updating $MYDIR/$STDERR_NAME
+        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    fi
+done
+
+
diff --git a/tests/ui/bad_toml/conf_bad_toml.stderr b/tests/ui/bad_toml/conf_bad_toml.stderr
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/ui/bad_toml_type/conf_bad_type.stderr b/tests/ui/bad_toml_type/conf_bad_type.stderr
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/ui/toml_blacklist/conf_french_blacklisted_name.stderr b/tests/ui/toml_blacklist/conf_french_blacklisted_name.stderr
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/ui/toml_unknown_key/conf_unknown_key.stderr b/tests/ui/toml_unknown_key/conf_unknown_key.stderr
deleted file mode 100644
index e69de29bb2d..00000000000