diff --git a/tests/compile-test.rs b/tests/compile-test.rs
index ff594f7a464..236cce0dbb7 100644
--- a/tests/compile-test.rs
+++ b/tests/compile-test.rs
@@ -3,9 +3,9 @@
 extern crate compiletest_rs as compiletest;
 extern crate test;
 
+use std::io;
 use std::ffi::OsStr;
 use std::fs;
-use std::error::Error;
 use std::env::{set_var, var};
 use std::path::{Path, PathBuf};
 
@@ -33,7 +33,7 @@ fn rustc_lib_path() -> PathBuf {
     option_env!("RUSTC_LIB_PATH").unwrap().into()
 }
 
-fn config(mode: &str, dir: &str) -> compiletest::Config {
+fn config(mode: &str, dir: PathBuf) -> compiletest::Config {
     let mut config = compiletest::Config::default();
 
     let cfg_mode = mode.parse().expect("Invalid mode");
@@ -59,25 +59,25 @@ fn config(mode: &str, dir: &str) -> compiletest::Config {
         path.push("target/debug/test_build_base");
         path
     };
-    config.src_base = PathBuf::from(format!("tests/{}", dir));
+    config.src_base = dir;
     config.rustc_path = clippy_driver_path();
     config
 }
 
-fn run_mode(mode: &str, dir: &str) {
+fn run_mode(mode: &str, dir: PathBuf) {
     compiletest::run_tests(&config(mode, dir));
 }
 
-fn run_ui_toml() -> Result<(), Box<Error>> {
-    let base = PathBuf::from("tests/ui-toml/").canonicalize()?;
-    for dir in fs::read_dir(&base)? {
+fn run_ui_toml_tests(config: &compiletest::Config, mut tests: Vec<test::TestDescAndFn>) -> Result<bool, io::Error> {
+    let mut result = true;
+    let opts = compiletest::test_opts(config);
+    for dir in fs::read_dir(&config.src_base)? {
         let dir = dir?;
         if !dir.file_type()?.is_dir() {
             continue;
         }
         let dir_path = dir.path();
         set_var("CARGO_MANIFEST_DIR", &dir_path);
-        let config = config("ui", "ui-toml");
         for file in fs::read_dir(&dir_path)? {
             let file = file?;
             let file_path = file.path();
@@ -89,13 +89,34 @@ fn run_ui_toml() -> Result<(), Box<Error>> {
             }
             let paths = compiletest::common::TestPaths {
                 file: file_path,
-                base: base.clone(),
+                base: config.src_base.clone(),
                 relative_dir: dir_path.file_name().unwrap().into(),
             };
-            compiletest::runtest::run(config.clone(), &paths);
+            let test_name = compiletest::make_test_name(&config, &paths);
+            let index = tests.iter()
+                .position(|test| test.desc.name == test_name)
+                .expect("The test should be in there");
+            result &= test::run_tests_console(
+                &opts,
+                vec![tests.swap_remove(index)])?;
+        }
+    }
+    Ok(result)
+}
+
+fn run_ui_toml() {
+    let path = PathBuf::from("tests/ui-toml").canonicalize().unwrap();
+    let config = config("ui", path);
+    let tests = compiletest::make_tests(&config);
+
+    let res = run_ui_toml_tests(&config, tests);
+    match res {
+        Ok(true) => {}
+        Ok(false) => panic!("Some tests failed"),
+        Err(e) => {
+            println!("I/O failure during tests: {:?}", e);
         }
     }
-    Ok(())
 }
 
 fn prepare_env() {
@@ -107,7 +128,7 @@ fn prepare_env() {
 #[test]
 fn compile_test() {
     prepare_env();
-    run_mode("run-pass", "run-pass");
-    run_mode("ui", "ui");
-    run_ui_toml().unwrap();
+    run_mode("run-pass", "tests/run-pass".into());
+    run_mode("ui", "tests/ui".into());
+    run_ui_toml();
 }