Auto merge of #38072 - nikomatsakis:bootstrap-incremental, r=acrichto
add preliminary support for incremental compilation to rustbuild.py This implements the integration described in #37929. It requires the use of a local nightly as your bootstrap compiler. The setup is described in `src/bootstrap/README.md`. This does NOT implement the "copy stage0 libs to stage1" optimization described in #37929, just because that seems orthogonal to me. In local testing, I do not yet see any incremental re-use when building rustc. I'm not sure why that is, more investigation needed. (For these reasons, this is not marked as fixing the relevant issue.) r? @alexcrichton -- I included one random cleanup (`Step::noop()`) that turned out to not be especially relevant. Feel free to tell me you liked it better the old way.
This commit is contained in:
commit
94ae2a2e67
|
@ -116,6 +116,42 @@ compiler. What actually happens when you invoke rustbuild is:
|
|||
The goal of each stage is to (a) leverage Cargo as much as possible and failing
|
||||
that (b) leverage Rust as much as possible!
|
||||
|
||||
## Incremental builds
|
||||
|
||||
You can configure rustbuild to use incremental compilation. Because
|
||||
incremental is new and evolving rapidly, if you want to use it, it is
|
||||
recommended that you replace the snapshot with a locally installed
|
||||
nightly build of rustc. You will want to keep this up to date.
|
||||
|
||||
To follow this course of action, first thing you will want to do is to
|
||||
install a nightly, presumably using `rustup`. You will then want to
|
||||
configure your directory to use this build, like so:
|
||||
|
||||
```
|
||||
# configure to use local rust instead of downloding a beta.
|
||||
# `--local-rust-root` is optional here. If elided, we will
|
||||
# use whatever rustc we find on your PATH.
|
||||
> configure --enable-rustbuild --local-rust-root=~/.cargo/ --enable-local-rebuild
|
||||
```
|
||||
|
||||
After that, you can use the `--incremental` flag to actually do
|
||||
incremental builds:
|
||||
|
||||
```
|
||||
> ../x.py build --incremental
|
||||
```
|
||||
|
||||
The `--incremental` flag will store incremental compilation artifacts
|
||||
in `build/stage0-incremental`. Note that we only use incremental
|
||||
compilation for the stage0 -> stage1 compilation -- this is because
|
||||
the stage1 compiler is changing, and we don't try to cache and reuse
|
||||
incremental artifacts across different versions of the compiler. For
|
||||
this reason, `--incremental` defaults to `--stage 1` (though you can
|
||||
manually select a higher stage, if you prefer).
|
||||
|
||||
You can always drop the `--incremental` to build as normal (but you
|
||||
will still be using the local nightly as your bootstrap).
|
||||
|
||||
## Directory Layout
|
||||
|
||||
This build system houses all output under the `build` directory, which looks
|
||||
|
|
|
@ -29,6 +29,9 @@ extern crate bootstrap;
|
|||
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::io;
|
||||
use std::io::prelude::*;
|
||||
use std::str::FromStr;
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Command, ExitStatus};
|
||||
|
||||
|
@ -41,6 +44,11 @@ fn main() {
|
|||
.and_then(|w| w[1].to_str());
|
||||
let version = args.iter().find(|w| &**w == "-vV");
|
||||
|
||||
let verbose = match env::var("RUSTC_VERBOSE") {
|
||||
Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
|
||||
Err(_) => 0,
|
||||
};
|
||||
|
||||
// Build scripts always use the snapshot compiler which is guaranteed to be
|
||||
// able to produce an executable, whereas intermediate compilers may not
|
||||
// have the standard library built yet and may not be able to produce an
|
||||
|
@ -95,6 +103,15 @@ fn main() {
|
|||
cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
// Pass down incremental directory, if any.
|
||||
if let Ok(dir) = env::var("RUSTC_INCREMENTAL") {
|
||||
cmd.arg(format!("-Zincremental={}", dir));
|
||||
|
||||
if verbose > 0 {
|
||||
cmd.arg("-Zincremental-info");
|
||||
}
|
||||
}
|
||||
|
||||
// If we're compiling specifically the `panic_abort` crate then we pass
|
||||
// the `-C panic=abort` option. Note that we do not do this for any
|
||||
// other crate intentionally as this is the only crate for now that we
|
||||
|
@ -176,8 +193,18 @@ fn main() {
|
|||
if let Some(rpath) = rpath {
|
||||
cmd.arg("-C").arg(format!("link-args={}", rpath));
|
||||
}
|
||||
|
||||
if let Ok(s) = env::var("RUSTFLAGS") {
|
||||
for flag in s.split_whitespace() {
|
||||
cmd.arg(flag);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if verbose > 1 {
|
||||
writeln!(&mut io::stderr(), "rustc command: {:?}", cmd).unwrap();
|
||||
}
|
||||
|
||||
// Actually run the compiler!
|
||||
std::process::exit(match exec_cmd(&mut cmd) {
|
||||
|
|
|
@ -294,6 +294,8 @@ class RustBuild(object):
|
|||
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
|
||||
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
|
||||
os.pathsep + env["PATH"]
|
||||
if not os.path.isfile(self.cargo()):
|
||||
raise Exception("no cargo executable found at `%s`" % self.cargo())
|
||||
args = [self.cargo(), "build", "--manifest-path",
|
||||
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
|
||||
if self.use_vendored_sources:
|
||||
|
|
|
@ -190,7 +190,7 @@ pub fn compiletest(build: &Build,
|
|||
|
||||
cmd.args(&build.flags.cmd.test_args());
|
||||
|
||||
if build.config.verbose || build.flags.verbose {
|
||||
if build.config.verbose() || build.flags.verbose() {
|
||||
cmd.arg("--verbose");
|
||||
}
|
||||
|
||||
|
|
|
@ -40,7 +40,7 @@ use util::push_exe_path;
|
|||
pub struct Config {
|
||||
pub ccache: Option<String>,
|
||||
pub ninja: bool,
|
||||
pub verbose: bool,
|
||||
pub verbose: usize,
|
||||
pub submodules: bool,
|
||||
pub compiler_docs: bool,
|
||||
pub docs: bool,
|
||||
|
@ -504,6 +504,14 @@ impl Config {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn verbose(&self) -> bool {
|
||||
self.verbose > 0
|
||||
}
|
||||
|
||||
pub fn very_verbose(&self) -> bool {
|
||||
self.verbose > 1
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
|
|
|
@ -27,7 +27,7 @@ use step;
|
|||
|
||||
/// Deserialized version of all flags for this compile.
|
||||
pub struct Flags {
|
||||
pub verbose: bool,
|
||||
pub verbose: usize, // verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
|
||||
pub stage: Option<u32>,
|
||||
pub keep_stage: Option<u32>,
|
||||
pub build: String,
|
||||
|
@ -37,6 +37,17 @@ pub struct Flags {
|
|||
pub src: Option<PathBuf>,
|
||||
pub jobs: Option<u32>,
|
||||
pub cmd: Subcommand,
|
||||
pub incremental: bool,
|
||||
}
|
||||
|
||||
impl Flags {
|
||||
pub fn verbose(&self) -> bool {
|
||||
self.verbose > 0
|
||||
}
|
||||
|
||||
pub fn very_verbose(&self) -> bool {
|
||||
self.verbose > 1
|
||||
}
|
||||
}
|
||||
|
||||
pub enum Subcommand {
|
||||
|
@ -63,7 +74,8 @@ pub enum Subcommand {
|
|||
impl Flags {
|
||||
pub fn parse(args: &[String]) -> Flags {
|
||||
let mut opts = Options::new();
|
||||
opts.optflag("v", "verbose", "use verbose output");
|
||||
opts.optflagmulti("v", "verbose", "use verbose output (-vv for very verbose)");
|
||||
opts.optflag("i", "incremental", "use incremental compilation");
|
||||
opts.optopt("", "config", "TOML configuration file for build", "FILE");
|
||||
opts.optopt("", "build", "build target of the stage0 compiler", "BUILD");
|
||||
opts.optmulti("", "host", "host targets to build", "HOST");
|
||||
|
@ -256,8 +268,18 @@ To learn more about a subcommand, run `./x.py <command> -h`
|
|||
}
|
||||
});
|
||||
|
||||
let mut stage = m.opt_str("stage").map(|j| j.parse().unwrap());
|
||||
|
||||
let incremental = m.opt_present("i");
|
||||
|
||||
if incremental {
|
||||
if stage.is_none() {
|
||||
stage = Some(1);
|
||||
}
|
||||
}
|
||||
|
||||
Flags {
|
||||
verbose: m.opt_present("v"),
|
||||
verbose: m.opt_count("v"),
|
||||
stage: m.opt_str("stage").map(|j| j.parse().unwrap()),
|
||||
keep_stage: m.opt_str("keep-stage").map(|j| j.parse().unwrap()),
|
||||
build: m.opt_str("build").unwrap_or_else(|| {
|
||||
|
@ -269,6 +291,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
|
|||
src: m.opt_str("src").map(PathBuf::from),
|
||||
jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()),
|
||||
cmd: cmd,
|
||||
incremental: incremental,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -74,6 +74,7 @@ extern crate rustc_serialize;
|
|||
extern crate toml;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::cmp;
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs::{self, File};
|
||||
|
@ -497,6 +498,17 @@ impl Build {
|
|||
cargo.env("RUSTC_BOOTSTRAP", "1");
|
||||
self.add_rust_test_threads(&mut cargo);
|
||||
|
||||
// Ignore incremental modes except for stage0, since we're
|
||||
// not guaranteeing correctness acros builds if the compiler
|
||||
// is changing under your feet.`
|
||||
if self.flags.incremental && compiler.stage == 0 {
|
||||
let incr_dir = self.incremental_dir(compiler);
|
||||
cargo.env("RUSTC_INCREMENTAL", incr_dir);
|
||||
}
|
||||
|
||||
let verbose = cmp::max(self.config.verbose, self.flags.verbose);
|
||||
cargo.env("RUSTC_VERBOSE", format!("{}", verbose));
|
||||
|
||||
// Specify some various options for build scripts used throughout
|
||||
// the build.
|
||||
//
|
||||
|
@ -516,7 +528,7 @@ impl Build {
|
|||
// FIXME: should update code to not require this env var
|
||||
cargo.env("CFG_COMPILER_HOST_TRIPLE", target);
|
||||
|
||||
if self.config.verbose || self.flags.verbose {
|
||||
if self.config.verbose() || self.flags.verbose() {
|
||||
cargo.arg("-v");
|
||||
}
|
||||
// FIXME: cargo bench does not accept `--release`
|
||||
|
@ -630,6 +642,12 @@ impl Build {
|
|||
}
|
||||
}
|
||||
|
||||
/// Get the directory for incremental by-products when using the
|
||||
/// given compiler.
|
||||
fn incremental_dir(&self, compiler: &Compiler) -> PathBuf {
|
||||
self.out.join(compiler.host).join(format!("stage{}-incremental", compiler.stage))
|
||||
}
|
||||
|
||||
/// Returns the libdir where the standard library and other artifacts are
|
||||
/// found for a compiler's sysroot.
|
||||
fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf {
|
||||
|
@ -768,7 +786,7 @@ impl Build {
|
|||
|
||||
/// Prints a message if this build is configured in verbose mode.
|
||||
fn verbose(&self, msg: &str) {
|
||||
if self.flags.verbose || self.config.verbose {
|
||||
if self.flags.verbose() || self.config.verbose() {
|
||||
println!("{}", msg);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -86,7 +86,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
//
|
||||
// To handle this we do a bit of dynamic dispatch to see what the dependency
|
||||
// is. If we're building a LLVM for the build triple, then we don't actually
|
||||
// have any dependencies! To do that we return a dependency on the "dummy"
|
||||
// have any dependencies! To do that we return a dependency on the `Step::noop()`
|
||||
// target which does nothing.
|
||||
//
|
||||
// If we're build a cross-compiled LLVM, however, we need to assemble the
|
||||
|
@ -104,7 +104,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
.host(true)
|
||||
.dep(move |s| {
|
||||
if s.target == build.config.build {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
} else {
|
||||
s.target(&build.config.build)
|
||||
}
|
||||
|
@ -115,14 +115,11 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
// going on here. You can check out the API docs below and also see a bunch
|
||||
// more examples of rules directly below as well.
|
||||
|
||||
// dummy rule to do nothing, useful when a dep maps to no deps
|
||||
rules.build("dummy", "path/to/nowhere");
|
||||
|
||||
// the compiler with no target libraries ready to go
|
||||
rules.build("rustc", "src/rustc")
|
||||
.dep(move |s| {
|
||||
if s.stage == 0 {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
} else {
|
||||
s.name("librustc")
|
||||
.host(&build.config.build)
|
||||
|
@ -165,7 +162,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
.dep(move |s| s.name("rustc").host(&build.config.build).target(s.host))
|
||||
.dep(move |s| {
|
||||
if s.host == build.config.build {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
} else {
|
||||
s.host(&build.config.build)
|
||||
}
|
||||
|
@ -183,7 +180,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
.dep(|s| s.name("libstd"))
|
||||
.dep(move |s| {
|
||||
if s.host == build.config.build {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
} else {
|
||||
s.host(&build.config.build)
|
||||
}
|
||||
|
@ -203,7 +200,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
.dep(move |s| s.name("llvm").host(&build.config.build).stage(0))
|
||||
.dep(move |s| {
|
||||
if s.host == build.config.build {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
} else {
|
||||
s.host(&build.config.build)
|
||||
}
|
||||
|
@ -233,7 +230,7 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
if s.target.contains("android") {
|
||||
s.name("android-copy-libs")
|
||||
} else {
|
||||
dummy(s, build)
|
||||
Step::noop()
|
||||
}
|
||||
})
|
||||
.default(true)
|
||||
|
@ -514,12 +511,6 @@ pub fn build_rules(build: &Build) -> Rules {
|
|||
|
||||
rules.verify();
|
||||
return rules;
|
||||
|
||||
fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> {
|
||||
s.name("dummy").stage(0)
|
||||
.target(&build.config.build)
|
||||
.host(&build.config.build)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
|
||||
|
@ -543,6 +534,10 @@ struct Step<'a> {
|
|||
}
|
||||
|
||||
impl<'a> Step<'a> {
|
||||
fn noop() -> Step<'a> {
|
||||
Step { name: "", stage: 0, host: "", target: "" }
|
||||
}
|
||||
|
||||
/// Creates a new step which is the same as this, except has a new name.
|
||||
fn name(&self, name: &'a str) -> Step<'a> {
|
||||
Step { name: name, ..*self }
|
||||
|
@ -738,6 +733,9 @@ impl<'a> Rules<'a> {
|
|||
if self.rules.contains_key(&dep.name) || dep.name.starts_with("default:") {
|
||||
continue
|
||||
}
|
||||
if dep == Step::noop() {
|
||||
continue
|
||||
}
|
||||
panic!("\
|
||||
|
||||
invalid rule dependency graph detected, was a rule added and maybe typo'd?
|
||||
|
@ -864,6 +862,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
|
|||
// of what we need to do.
|
||||
let mut order = Vec::new();
|
||||
let mut added = HashSet::new();
|
||||
added.insert(Step::noop());
|
||||
for step in steps.iter().cloned() {
|
||||
self.fill(step, &mut order, &mut added);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue