Auto merge of #39329 - petrochenkov:rb2, r=alexcrichton

rustbuild: Build jemalloc and libbacktrace only once (take 2)

This is a rebase of https://github.com/rust-lang/rust/pull/38583 without any additions, but with implemented @alexcrichton's suggestions.
~~This includes `exists(Makefile)` => `cfg(stage0)` suggestion... but it will break cross-compilation, no? Are `libstd/liballoc_jemalloc` cross-compiled for `target != host` built during `stage0`?~~

r? @alexcrichton

src/Cargo.lock

@@ -61,6 +61,9 @@ dependencies = [
 [[package]]
 name = "build_helper"
 version = "0.1.0"
+dependencies = [
+ "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "cargotest"
@@ -594,7 +597,7 @@ dependencies = [
 
 [[package]]
 name = "std_shim"
-version = "0.1.0"
+version = "0.0.0"
 dependencies = [
  "core 0.0.0",
  "std 0.0.0",
@@ -653,7 +656,7 @@ dependencies = [
 
 [[package]]
 name = "test_shim"
-version = "0.1.0"
+version = "0.0.0"
 dependencies = [
  "test 0.0.0",
 ]

src/bootstrap/check.rs

@@ -383,7 +383,7 @@ pub fn krate(build: &Build,
                 // helper crate, not tested. If it leaks through then it ends up
                 // messing with various mtime calculations and such.
                 if !name.contains("jemalloc") && name != "build_helper" {
-                    cargo.arg("-p").arg(name);
+                    cargo.arg("-p").arg(&format!("{}:0.0.0", name));
                 }
                 for dep in build.crates[name].deps.iter() {
                     if visited.insert(dep) {

src/bootstrap/compile.rs

@@ -21,10 +21,10 @@ use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use build_helper::output;
+use build_helper::{output, mtime};
 use filetime::FileTime;
 
-use util::{exe, libdir, mtime, is_dylib, copy};
+use util::{exe, libdir, is_dylib, copy};
 use {Build, Compiler, Mode};
 
 /// Build the standard library.

src/bootstrap/doc.rs

@@ -22,7 +22,8 @@ use std::io::prelude::*;
 use std::process::Command;
 
 use {Build, Compiler, Mode};
-use util::{up_to_date, cp_r};
+use util::cp_r;
+use build_helper::up_to_date;
 
 /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
 /// `name` into the `out` path.

src/bootstrap/lib.rs

@@ -66,6 +66,7 @@
 
 #![deny(warnings)]
 
+#[macro_use]
 extern crate build_helper;
 extern crate cmake;
 extern crate filetime;
@@ -83,24 +84,9 @@ use std::fs::{self, File};
 use std::path::{Component, PathBuf, Path};
 use std::process::Command;
 
-use build_helper::{run_silent, output};
+use build_helper::{run_silent, output, mtime};
 
-use util::{exe, mtime, libdir, add_lib_path};
+use util::{exe, libdir, add_lib_path};
-
-/// A helper macro to `unwrap` a result except also print out details like:
-///
-/// * The file/line of the panic
-/// * The expression that failed
-/// * The error itself
-///
-/// This is currently used judiciously throughout the build system rather than
-/// using a `Result` with `try!`, but this may change one day...
-macro_rules! t {
-    ($e:expr) => (match $e {
-        Ok(e) => e,
-        Err(e) => panic!("{} failed with {}", stringify!($e), e),
-    })
-}
 
 mod cc;
 mod channel;
@@ -482,7 +468,8 @@ impl Build {
         //
         // These variables are primarily all read by
         // src/bootstrap/bin/{rustc.rs,rustdoc.rs}
-        cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc"))
+        cargo.env("RUSTBUILD_NATIVE_DIR", self.native_dir(target))
+             .env("RUSTC", self.out.join("bootstrap/debug/rustc"))
              .env("RUSTC_REAL", self.compiler_path(compiler))
              .env("RUSTC_STAGE", stage.to_string())
              .env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
@@ -746,10 +733,15 @@ impl Build {
         }
     }
 
+    /// Directory for libraries built from C/C++ code and shared between stages.
+    fn native_dir(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("native")
+    }
+
     /// Root output directory for rust_test_helpers library compiled for
     /// `target`
     fn test_helpers_out(&self, target: &str) -> PathBuf {
-        self.out.join(target).join("rust-test-helpers")
+        self.native_dir(target).join("rust-test-helpers")
     }
 
     /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic

src/bootstrap/native.rs

@@ -28,7 +28,8 @@ use cmake;
 use gcc;
 
 use Build;
-use util::{self, up_to_date};
+use util;
+use build_helper::up_to_date;
 
 /// Compile LLVM for `target`.
 pub fn llvm(build: &Build, target: &str) {

src/bootstrap/util.rs

@@ -20,8 +20,6 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::Instant;
 
-use filetime::FileTime;
-
 /// Returns the `name` as the filename of a static library for `target`.
 pub fn staticlib(name: &str, target: &str) -> String {
     if target.contains("windows") {
@@ -31,13 +29,6 @@ pub fn staticlib(name: &str, target: &str) -> String {
     }
 }
 
-/// Returns the last-modified time for `path`, or zero if it doesn't exist.
-pub fn mtime(path: &Path) -> FileTime {
-    fs::metadata(path).map(|f| {
-        FileTime::from_last_modification_time(&f)
-    }).unwrap_or(FileTime::zero())
-}
-
 /// Copies a file from `src` to `dst`, attempting to use hard links and then
 /// falling back to an actually filesystem copy if necessary.
 pub fn copy(src: &Path, dst: &Path) {
@@ -132,34 +123,6 @@ pub fn add_lib_path(path: Vec<PathBuf>, cmd: &mut Command) {
     cmd.env(dylib_path_var(), t!(env::join_paths(list)));
 }
 
-/// Returns whether `dst` is up to date given that the file or files in `src`
-/// are used to generate it.
-///
-/// Uses last-modified time checks to verify this.
-pub fn up_to_date(src: &Path, dst: &Path) -> bool {
-    let threshold = mtime(dst);
-    let meta = match fs::metadata(src) {
-        Ok(meta) => meta,
-        Err(e) => panic!("source {:?} failed to get metadata: {}", src, e),
-    };
-    if meta.is_dir() {
-        dir_up_to_date(src, &threshold)
-    } else {
-        FileTime::from_last_modification_time(&meta) <= threshold
-    }
-}
-
-fn dir_up_to_date(src: &Path, threshold: &FileTime) -> bool {
-    t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
-        let meta = t!(e.metadata());
-        if meta.is_dir() {
-            dir_up_to_date(&e.path(), threshold)
-        } else {
-            FileTime::from_last_modification_time(&meta) < *threshold
-        }
-    })
-}
-
 /// Returns the environment variable which the dynamic library lookup path
 /// resides in for this platform.
 pub fn dylib_path_var() -> &'static str {

src/build_helper/Cargo.toml

@@ -6,3 +6,6 @@ authors = ["The Rust Project Developers"]
 [lib]
 name = "build_helper"
 path = "lib.rs"
+
+[dependencies]
+filetime = "0.1"

src/build_helper/lib.rs

@@ -10,9 +10,30 @@
 
 #![deny(warnings)]
 
+extern crate filetime;
+
+use std::fs;
 use std::process::{Command, Stdio};
 use std::path::{Path, PathBuf};
 
+use filetime::FileTime;
+
+/// A helper macro to `unwrap` a result except also print out details like:
+///
+/// * The file/line of the panic
+/// * The expression that failed
+/// * The error itself
+///
+/// This is currently used judiciously throughout the build system rather than
+/// using a `Result` with `try!`, but this may change one day...
+#[macro_export]
+macro_rules! t {
+    ($e:expr) => (match $e {
+        Ok(e) => e,
+        Err(e) => panic!("{} failed with {}", stringify!($e), e),
+    })
+}
+
 pub fn run(cmd: &mut Command) {
     println!("running: {:?}", cmd);
     run_silent(cmd);
@@ -88,6 +109,56 @@ pub fn output(cmd: &mut Command) -> String {
     String::from_utf8(output.stdout).unwrap()
 }
 
+pub fn rerun_if_changed_anything_in_dir(dir: &Path) {
+    let mut stack = dir.read_dir().unwrap()
+                       .map(|e| e.unwrap())
+                       .filter(|e| &*e.file_name() != ".git")
+                       .collect::<Vec<_>>();
+    while let Some(entry) = stack.pop() {
+        let path = entry.path();
+        if entry.file_type().unwrap().is_dir() {
+            stack.extend(path.read_dir().unwrap().map(|e| e.unwrap()));
+        } else {
+            println!("cargo:rerun-if-changed={}", path.display());
+        }
+    }
+}
+
+/// Returns the last-modified time for `path`, or zero if it doesn't exist.
+pub fn mtime(path: &Path) -> FileTime {
+    fs::metadata(path).map(|f| {
+        FileTime::from_last_modification_time(&f)
+    }).unwrap_or(FileTime::zero())
+}
+
+/// Returns whether `dst` is up to date given that the file or files in `src`
+/// are used to generate it.
+///
+/// Uses last-modified time checks to verify this.
+pub fn up_to_date(src: &Path, dst: &Path) -> bool {
+    let threshold = mtime(dst);
+    let meta = match fs::metadata(src) {
+        Ok(meta) => meta,
+        Err(e) => panic!("source {:?} failed to get metadata: {}", src, e),
+    };
+    if meta.is_dir() {
+        dir_up_to_date(src, &threshold)
+    } else {
+        FileTime::from_last_modification_time(&meta) <= threshold
+    }
+}
+
+fn dir_up_to_date(src: &Path, threshold: &FileTime) -> bool {
+    t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
+        let meta = t!(e.metadata());
+        if meta.is_dir() {
+            dir_up_to_date(&e.path(), threshold)
+        } else {
+            FileTime::from_last_modification_time(&meta) < *threshold
+        }
+    })
+}
+
 fn fail(s: &str) -> ! {
     println!("\n\n{}\n\n", s);
     std::process::exit(1);

src/liballoc_jemalloc/build.rs

@@ -10,23 +10,20 @@
 
 #![deny(warnings)]
 
+#[macro_use]
 extern crate build_helper;
 extern crate gcc;
 
 use std::env;
-use std::path::PathBuf;
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
 use std::process::Command;
-use build_helper::run;
+use build_helper::{run, rerun_if_changed_anything_in_dir, up_to_date};
 
 fn main() {
     println!("cargo:rustc-cfg=cargobuild");
     println!("cargo:rerun-if-changed=build.rs");
 
-    let target = env::var("TARGET").expect("TARGET was not set");
-    let host = env::var("HOST").expect("HOST was not set");
-    let build_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-    let src_dir = env::current_dir().unwrap();
-
     // FIXME: This is a hack to support building targets that don't
     // support jemalloc alongside hosts that do. The jemalloc build is
     // controlled by a feature of the std crate, and if that feature
@@ -35,6 +32,8 @@ fn main() {
     // that the feature set used by std is the same across all
     // targets, which means we have to build the alloc_jemalloc crate
     // for targets like emscripten, even if we don't use it.
+    let target = env::var("TARGET").expect("TARGET was not set");
+    let host = env::var("HOST").expect("HOST was not set");
     if target.contains("rumprun") || target.contains("bitrig") || target.contains("openbsd") ||
        target.contains("msvc") || target.contains("emscripten") || target.contains("fuchsia") ||
        target.contains("redox") {
@@ -57,6 +56,28 @@ fn main() {
         return;
     }
 
+    let build_dir = env::var_os("RUSTBUILD_NATIVE_DIR").unwrap_or(env::var_os("OUT_DIR").unwrap());
+    let build_dir = PathBuf::from(build_dir).join("jemalloc");
+    let _ = fs::create_dir_all(&build_dir);
+
+    if target.contains("windows") {
+        println!("cargo:rustc-link-lib=static=jemalloc");
+    } else {
+        println!("cargo:rustc-link-lib=static=jemalloc_pic");
+    }
+    println!("cargo:rustc-link-search=native={}/lib", build_dir.display());
+    if target.contains("android") {
+        println!("cargo:rustc-link-lib=gcc");
+    } else if !target.contains("windows") && !target.contains("musl") {
+        println!("cargo:rustc-link-lib=pthread");
+    }
+    let src_dir = env::current_dir().unwrap().join("../jemalloc");
+    rerun_if_changed_anything_in_dir(&src_dir);
+    let timestamp = build_dir.join("rustbuild.timestamp");
+    if up_to_date(&Path::new("build.rs"), &timestamp) && up_to_date(&src_dir, &timestamp) {
+        return
+    }
+
     let compiler = gcc::Config::new().get_compiler();
     // only msvc returns None for ar so unwrap is okay
     let ar = build_helper::cc2ar(compiler.path(), &target).unwrap();
@@ -66,23 +87,8 @@ fn main() {
         .collect::<Vec<_>>()
         .join(" ");
 
-    let mut stack = src_dir.join("../jemalloc")
-        .read_dir()
-        .unwrap()
-        .map(|e| e.unwrap())
-        .filter(|e| &*e.file_name() != ".git")
-        .collect::<Vec<_>>();
-    while let Some(entry) = stack.pop() {
-        let path = entry.path();
-        if entry.file_type().unwrap().is_dir() {
-            stack.extend(path.read_dir().unwrap().map(|e| e.unwrap()));
-        } else {
-            println!("cargo:rerun-if-changed={}", path.display());
-        }
-    }
-
     let mut cmd = Command::new("sh");
-    cmd.arg(src_dir.join("../jemalloc/configure")
+    cmd.arg(src_dir.join("configure")
                    .to_str()
                    .unwrap()
                    .replace("C:\\", "/c/")
@@ -158,6 +164,7 @@ fn main() {
     }
 
     run(&mut cmd);
+
     let mut make = Command::new(build_helper::make(&host));
     make.current_dir(&build_dir)
         .arg("build_lib_static");
@@ -170,18 +177,6 @@ fn main() {
 
     run(&mut make);
 
-    if target.contains("windows") {
-        println!("cargo:rustc-link-lib=static=jemalloc");
-    } else {
-        println!("cargo:rustc-link-lib=static=jemalloc_pic");
-    }
-    println!("cargo:rustc-link-search=native={}/lib", build_dir.display());
-    if target.contains("android") {
-        println!("cargo:rustc-link-lib=gcc");
-    } else if !target.contains("windows") && !target.contains("musl") {
-        println!("cargo:rustc-link-lib=pthread");
-    }
-
     // The pthread_atfork symbols is used by jemalloc on android but the really
     // old android we're building on doesn't have them defined, so just make
     // sure the symbols are available.
@@ -192,4 +187,6 @@ fn main() {
             .file("pthread_atfork_dummy.c")
             .compile("libpthread_atfork_dummy.a");
     }
+
+    t!(File::create(&timestamp));
 }

src/libstd/build.rs

@@ -10,14 +10,15 @@
 
 #![deny(warnings)]
 
-extern crate gcc;
+#[macro_use]
 extern crate build_helper;
+extern crate gcc;
 
 use std::env;
-use std::path::PathBuf;
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
 use std::process::Command;
-
+use build_helper::{run, rerun_if_changed_anything_in_dir, up_to_date};
-use build_helper::run;
 
 fn main() {
     println!("cargo:rustc-cfg=cargobuild");
@@ -66,22 +67,17 @@ fn main() {
 }
 
 fn build_libbacktrace(host: &str, target: &str) {
-    let src_dir = env::current_dir().unwrap().join("../libbacktrace");
+    let build_dir = env::var_os("RUSTBUILD_NATIVE_DIR").unwrap_or(env::var_os("OUT_DIR").unwrap());
-    let build_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
+    let build_dir = PathBuf::from(build_dir).join("libbacktrace");
+    let _ = fs::create_dir_all(&build_dir);
 
     println!("cargo:rustc-link-lib=static=backtrace");
     println!("cargo:rustc-link-search=native={}/.libs", build_dir.display());
-
+    let src_dir = env::current_dir().unwrap().join("../libbacktrace");
-    let mut stack = src_dir.read_dir().unwrap()
+    rerun_if_changed_anything_in_dir(&src_dir);
-                           .map(|e| e.unwrap())
+    let timestamp = build_dir.join("rustbuild.timestamp");
-                           .collect::<Vec<_>>();
+    if up_to_date(&Path::new("build.rs"), &timestamp) && up_to_date(&src_dir, &timestamp) {
-    while let Some(entry) = stack.pop() {
+        return
-        let path = entry.path();
-        if entry.file_type().unwrap().is_dir() {
-            stack.extend(path.read_dir().unwrap().map(|e| e.unwrap()));
-        } else {
-            println!("cargo:rerun-if-changed={}", path.display());
-        }
     }
 
     let compiler = gcc::Config::new().get_compiler();
@@ -105,8 +101,11 @@ fn build_libbacktrace(host: &str, target: &str) {
                 .env("AR", &ar)
                 .env("RANLIB", format!("{} s", ar.display()))
                 .env("CFLAGS", cflags));
+
     run(Command::new(build_helper::make(host))
                 .current_dir(&build_dir)
                 .arg(format!("INCDIR={}", src_dir.display()))
                 .arg("-j").arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set")));
+
+    t!(File::create(&timestamp));
 }

src/rustc/std_shim/Cargo.toml

@@ -21,7 +21,7 @@
 
 [package]
 name = "std_shim"
-version = "0.1.0"
+version = "0.0.0"
 authors = ["The Rust Project Developers"]
 
 [lib]

src/rustc/test_shim/Cargo.toml

@@ -5,7 +5,7 @@
 
 [package]
 name = "test_shim"
-version = "0.1.0"
+version = "0.0.0"
 authors = ["The Rust Project Developers"]
 
 [lib]