rustbuild: Add support for crate tests + doctests

This commit adds support to rustbuild to run crate unit tests (those defined by
`#[test]`) as well as documentation tests. All tests are powered by `cargo test`
under the hood.

Each step requires the `libtest` library is built for that corresponding stage.
Ideally the `test` crate would be a dev-dependency, but for now it's just easier
to ensure that we sequence everything in the right order.

Currently no filtering is implemented, so there's not actually a method of
testing *only* libstd or *only* libcore, but rather entire swaths of crates are
tested all at once.

A few points of note here are:

* The `coretest` and `collectionstest` crates are just listed as `[[test]]`
  entires for `cargo test` to naturally pick up. This mean that `cargo test -p
  core` actually runs all the tests for libcore.
* Libraries that aren't tested all mention `test = false` in their `Cargo.toml`
* Crates aren't currently allowed to have dev-dependencies due to
  rust-lang/cargo#860, but we can likely alleviate this restriction once
  workspaces are implemented.

cc #31590

src/bootstrap/build/check.rs

@@ -13,13 +13,16 @@
 //! This file implements the various regression test suites that we execute on
 //! our CI.
 
-use std::fs;
+use std::env;
+use std::fs::{self, File};
+use std::io::prelude::*;
 use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::output;
+use bootstrap::{dylib_path, dylib_path_var};
 
-use build::{Build, Compiler};
+use build::{Build, Compiler, Mode};
 
 /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 ///
@@ -222,3 +225,75 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
     cmd.arg("--test-args").arg(build.flags.args.join(" "));
     build.run(&mut cmd);
 }
+
+/// Run all unit tests plus documentation tests for an entire crate DAG defined
+/// by a `Cargo.toml`
+///
+/// This is what runs tests for crates like the standard library, compiler, etc.
+/// It essentially is the driver for running `cargo test`.
+///
+/// Currently this runs all tests for a DAG by passing a bunch of `-p foo`
+/// arguments, and those arguments are discovered from `Cargo.lock`.
+pub fn krate(build: &Build,
+             compiler: &Compiler,
+             target: &str,
+             mode: Mode) {
+    let (name, path, features) = match mode {
+        Mode::Libstd => ("libstd", "src/rustc/std_shim", build.std_features()),
+        Mode::Libtest => ("libtest", "src/rustc/test_shim", String::new()),
+        Mode::Librustc => ("librustc", "src/rustc", build.rustc_features()),
+        _ => panic!("can only test libraries"),
+    };
+    println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
+             compiler.host, target);
+
+    // Build up the base `cargo test` command.
+    let mut cargo = build.cargo(compiler, mode, target, "test");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join(path).join("Cargo.toml"))
+         .arg("--features").arg(features);
+
+    // Generate a list of `-p` arguments to pass to the `cargo test` invocation
+    // by crawling the corresponding Cargo.lock file.
+    let lockfile = build.src.join(path).join("Cargo.lock");
+    let mut contents = String::new();
+    t!(t!(File::open(&lockfile)).read_to_string(&mut contents));
+    let mut lines = contents.lines();
+    while let Some(line) = lines.next() {
+        let prefix = "name = \"";
+        if !line.starts_with(prefix) {
+            continue
+        }
+        lines.next(); // skip `version = ...`
+
+        // skip crates.io or otherwise non-path crates
+        if let Some(line) = lines.next() {
+            if line.starts_with("source") {
+                continue
+            }
+        }
+
+        let crate_name = &line[prefix.len()..line.len() - 1];
+
+        // Right now jemalloc is our only target-specific crate in the sense
+        // that it's not present on all platforms. Custom skip it here for now,
+        // but if we add more this probably wants to get more generalized.
+        if crate_name.contains("jemalloc") {
+            continue
+        }
+
+        cargo.arg("-p").arg(crate_name);
+    }
+
+    // The tests are going to run with the *target* libraries, so we need to
+    // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
+    //
+    // Note that to run the compiler we need to run with the *host* libraries,
+    // but our wrapper scripts arrange for that to be the case anyway.
+    let mut dylib_path = dylib_path();
+    dylib_path.insert(0, build.sysroot_libdir(compiler, target));
+    cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+    cargo.args(&build.flags.args);
+
+    build.run(&mut cargo);
+}

src/bootstrap/build/mod.rs

@@ -380,6 +380,15 @@ impl Build {
                     check::compiletest(self, &compiler, target.target,
                                        "run-make", "run-make")
                 }
+                CheckCrateStd { compiler } => {
+                    check::krate(self, &compiler, target.target, Mode::Libstd)
+                }
+                CheckCrateTest { compiler } => {
+                    check::krate(self, &compiler, target.target, Mode::Libtest)
+                }
+                CheckCrateRustc { compiler } => {
+                    check::krate(self, &compiler, target.target, Mode::Librustc)
+                }
 
                 DistDocs { stage } => dist::docs(self, stage, target.target),
                 DistMingw { _dummy } => dist::mingw(self, target.target),
@@ -485,6 +494,7 @@ impl Build {
                   self.config.rust_debug_assertions.to_string())
              .env("RUSTC_SNAPSHOT", &self.rustc)
              .env("RUSTC_SYSROOT", self.sysroot(compiler))
+             .env("RUSTC_LIBDIR", self.rustc_libdir(compiler))
              .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir())
              .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
              .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
@@ -520,7 +530,6 @@ impl Build {
         if self.config.rust_optimize {
             cargo.arg("--release");
         }
-        self.add_rustc_lib_path(compiler, &mut cargo);
         return cargo
     }
 

src/bootstrap/build/step.rs

@@ -120,6 +120,9 @@ macro_rules! targets {
             (check_docs, CheckDocs { compiler: Compiler<'a> }),
             (check_error_index, CheckErrorIndex { compiler: Compiler<'a> }),
             (check_rmake, CheckRMake { compiler: Compiler<'a> }),
+            (check_crate_std, CheckCrateStd { compiler: Compiler<'a> }),
+            (check_crate_test, CheckCrateTest { compiler: Compiler<'a> }),
+            (check_crate_rustc, CheckCrateRustc { compiler: Compiler<'a> }),
 
             // Distribution targets, creating tarballs
             (dist, Dist { stage: u32 }),
@@ -376,6 +379,9 @@ impl<'a> Step<'a> {
                     self.check_cfail(compiler),
                     self.check_rfail(compiler),
                     self.check_pfail(compiler),
+                    self.check_crate_std(compiler),
+                    self.check_crate_test(compiler),
+                    self.check_crate_rustc(compiler),
                     self.check_codegen(compiler),
                     self.check_codegen_units(compiler),
                     self.check_debuginfo(compiler),
@@ -437,6 +443,15 @@ impl<'a> Step<'a> {
             Source::CheckErrorIndex { compiler } => {
                 vec![self.libstd(compiler), self.tool_error_index(compiler.stage)]
             }
+            Source::CheckCrateStd { compiler } => {
+                vec![self.libtest(compiler)]
+            }
+            Source::CheckCrateTest { compiler } => {
+                vec![self.libtest(compiler)]
+            }
+            Source::CheckCrateRustc { compiler } => {
+                vec![self.libtest(compiler)]
+            }
 
             Source::ToolLinkchecker { stage } |
             Source::ToolTidy { stage } => {

src/bootstrap/rustc.rs

@@ -29,6 +29,7 @@ extern crate bootstrap;
 
 use std::env;
 use std::ffi::OsString;
+use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
@@ -43,16 +44,22 @@ fn main() {
     // have the standard library built yet and may not be able to produce an
     // executable. Otherwise we just use the standard compiler we're
     // bootstrapping with.
-    let rustc = if target.is_none() {
-        env::var_os("RUSTC_SNAPSHOT").unwrap()
+    let (rustc, libdir) = if target.is_none() {
+        ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR")
     } else {
-        env::var_os("RUSTC_REAL").unwrap()
+        ("RUSTC_REAL", "RUSTC_LIBDIR")
     };
     let stage = env::var("RUSTC_STAGE").unwrap();
 
+    let rustc = env::var_os(rustc).unwrap();
+    let libdir = env::var_os(libdir).unwrap();
+    let mut dylib_path = bootstrap::dylib_path();
+    dylib_path.insert(0, PathBuf::from(libdir));
+
     let mut cmd = Command::new(rustc);
     cmd.args(&args)
-       .arg("--cfg").arg(format!("stage{}", stage));
+       .arg("--cfg").arg(format!("stage{}", stage))
+       .env(bootstrap::dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
     if let Some(target) = target {
         // The stage0 compiler has a special sysroot distinct from what we

src/bootstrap/rustdoc.rs

@@ -12,17 +12,25 @@
 //!
 //! See comments in `src/bootstrap/rustc.rs` for more information.
 
+extern crate bootstrap;
+
 use std::env;
 use std::process::Command;
+use std::path::PathBuf;
 
 fn main() {
     let args = env::args_os().skip(1).collect::<Vec<_>>();
     let rustdoc = env::var_os("RUSTDOC_REAL").unwrap();
+    let libdir = env::var_os("RUSTC_LIBDIR").unwrap();
+
+    let mut dylib_path = bootstrap::dylib_path();
+    dylib_path.insert(0, PathBuf::from(libdir));
 
     let mut cmd = Command::new(rustdoc);
     cmd.args(&args)
        .arg("--cfg").arg(format!("stage{}", env::var("RUSTC_STAGE").unwrap()))
-       .arg("--cfg").arg("dox");
+       .arg("--cfg").arg("dox")
+       .env(bootstrap::dylib_path_var(), env::join_paths(&dylib_path).unwrap());
     std::process::exit(match cmd.status() {
         Ok(s) => s.code().unwrap_or(1),
         Err(e) => panic!("\n\nfailed to run {:?}: {}\n\n", cmd, e),

src/liballoc/Cargo.toml

@@ -6,7 +6,6 @@ version = "0.0.0"
 [lib]
 name = "alloc"
 path = "lib.rs"
-test = false
 
 [dependencies]
 core = { path = "../libcore" }

src/libcollections/Cargo.toml

@@ -6,9 +6,12 @@ version = "0.0.0"
 [lib]
 name = "collections"
 path = "lib.rs"
-test = false
 
 [dependencies]
 alloc = { path = "../liballoc" }
 core = { path = "../libcore" }
 rustc_unicode = { path = "../librustc_unicode" }
+
+[[test]]
+name = "collectionstest"
+path = "../libcollectionstest/lib.rs"

src/libcore/Cargo.toml

@@ -8,3 +8,7 @@ build = "build.rs"
 name = "core"
 path = "lib.rs"
 test = false
+
+[[test]]
+name = "coretest"
+path = "../libcoretest/lib.rs"

src/libflate/lib.rs

@@ -27,11 +27,7 @@
 #![feature(libc)]
 #![feature(staged_api)]
 #![feature(unique)]
-#![cfg_attr(test, feature(rustc_private, rand))]
-
-#[cfg(test)]
-#[macro_use]
-extern crate log;
+#![cfg_attr(test, feature(rand))]
 
 extern crate libc;
 
@@ -175,14 +171,8 @@ mod tests {
             for _ in 0..2000 {
                 input.extend_from_slice(r.choose(&words).unwrap());
             }
-            debug!("de/inflate of {} bytes of random word-sequences",
-                   input.len());
             let cmp = deflate_bytes(&input);
             let out = inflate_bytes(&cmp).unwrap();
-            debug!("{} bytes deflated to {} ({:.1}% size)",
-                   input.len(),
-                   cmp.len(),
-                   100.0 * ((cmp.len() as f64) / (input.len() as f64)));
             assert_eq!(&*input, &*out);
         }
     }

src/libpanic_abort/Cargo.toml

@@ -5,6 +5,7 @@ version = "0.0.0"
 
 [lib]
 path = "lib.rs"
+test = false
 
 [dependencies]
 core = { path = "../libcore" }

src/libpanic_unwind/Cargo.toml

@@ -5,6 +5,7 @@ version = "0.0.0"
 
 [lib]
 path = "lib.rs"
+test = false
 
 [dependencies]
 alloc = { path = "../liballoc" }

src/librand/Cargo.toml

@@ -6,7 +6,6 @@ version = "0.0.0"
 [lib]
 name = "rand"
 path = "lib.rs"
-test = false
 
 [dependencies]
 core = { path = "../libcore" }

src/librustc_bitflags/Cargo.toml

@@ -7,3 +7,4 @@ version = "0.0.0"
 name = "rustc_bitflags"
 path = "lib.rs"
 test = false
+doctest = false

src/librustc_borrowck/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_borrowck"
 path = "lib.rs"
 crate-type = ["dylib"]
+test = false
 
 [dependencies]
 log = { path = "../liblog" }

src/librustc_lint/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_lint"
 path = "lib.rs"
 crate-type = ["dylib"]
+test = false
 
 [dependencies]
 log = { path = "../liblog" }

src/librustc_resolve/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_resolve"
 path = "lib.rs"
 crate-type = ["dylib"]
+test = false
 
 [dependencies]
 log = { path = "../liblog" }

src/librustc_trans/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_trans"
 path = "lib.rs"
 crate-type = ["dylib"]
+test = false
 
 [dependencies]
 arena = { path = "../libarena" }

src/librustc_typeck/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_typeck"
 path = "lib.rs"
 crate-type = ["dylib"]
+test = false
 
 [dependencies]
 log = { path = "../liblog" }

src/libstd/Cargo.toml

@@ -8,7 +8,6 @@ build = "build.rs"
 name = "std"
 path = "lib.rs"
 crate-type = ["dylib", "rlib"]
-test = false
 
 [dependencies]
 alloc = { path = "../liballoc" }

src/libunwind/Cargo.toml

@@ -7,6 +7,7 @@ build = "build.rs"
 [lib]
 name = "unwind"
 path = "lib.rs"
+test = false
 
 [dependencies]
 core = { path = "../libcore" }

src/rustc/Cargo.toml

@@ -13,6 +13,8 @@ path = "rustdoc.rs"
 
 [profile.release]
 opt-level = 2
+[profile.bench]
+opt-level = 2
 
 # These options are controlled from our rustc wrapper script, so turn them off
 # here and have them controlled elsewhere.

src/rustc/libc_shim/Cargo.toml

@@ -15,6 +15,7 @@ build = "build.rs"
 [lib]
 name = "libc"
 path = "../../liblibc/src/lib.rs"
+test = false
 
 [dependencies]
 core = { path = "../../libcore" }

src/rustc/std_shim/Cargo.toml

@@ -30,6 +30,8 @@ path = "lib.rs"
 
 [profile.release]
 opt-level = 2
+[profile.bench]
+opt-level = 2
 
 # These options are controlled from our rustc wrapper script, so turn them off
 # here and have them controlled elsewhere.

src/rustc/test_shim/Cargo.toml

@@ -14,6 +14,8 @@ path = "lib.rs"
 
 [profile.release]
 opt-level = 2
+[profile.bench]
+opt-level = 2
 
 # These options are controlled from our rustc wrapper script, so turn them off
 # here and have them controlled elsewhere.