Merge branch 'new-rustbuild' into rollup

This commit is contained in:
Alex Crichton 2016-11-04 16:53:48 -07:00
commit 0eff43ff75
17 changed files with 1162 additions and 1011 deletions

View File

@ -127,7 +127,7 @@ ones from MSYS if you have it installed). You'll also need Visual Studio 2013 or
newer with the C++ tools. Then all you need to do is to kick off rustbuild.
```
python .\src\bootstrap\bootstrap.py
python x.py build
```
Currently rustbuild only works with some known versions of Visual Studio. If you
@ -137,7 +137,7 @@ by manually calling the appropriate vcvars file before running the bootstrap.
```
CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
python .\src\bootstrap\bootstrap.py
python x.py build
```
## Building Documentation

20
src/Cargo.lock generated
View File

@ -40,9 +40,9 @@ name = "bootstrap"
version = "0.0.0"
dependencies = [
"build_helper 0.1.0",
"cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
"cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)",
"gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
"getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
@ -63,7 +63,7 @@ version = "0.1.0"
[[package]]
name = "cmake"
version = "0.1.17"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
@ -131,11 +131,6 @@ dependencies = [
name = "fmt_macros"
version = "0.0.0"
[[package]]
name = "gcc"
version = "0.3.38"
source = "git+https://github.com/alexcrichton/gcc-rs#be620ac6d3ddb498cd0c700d5312c6a4c3c19597"
[[package]]
name = "gcc"
version = "0.3.38"
@ -189,7 +184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
name = "linkchecker"
version = "0.1.0"
dependencies = [
"url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -725,7 +720,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "url"
version = "1.2.2"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -743,10 +738,9 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dfcf5bcece56ef953b8ea042509e9dcbdfe97820b7e20d86beb53df30ed94978"
"checksum cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0e5bcf27e097a184c1df4437654ed98df3d7a516e8508a6ba45d8b092bbdf283"
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
"checksum gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)" = "<none>"
"checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5"
"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
@ -760,6 +754,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
"checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
"checksum url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba5a45db1d2e0effb7a1c00cc73ffc63a973da8c7d1fcd5b46f24285ade6c54"
"checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"

View File

@ -27,7 +27,7 @@ num_cpus = "0.2"
toml = "0.1"
getopts = "0.2"
rustc-serialize = "0.3"
gcc = { git = "https://github.com/alexcrichton/gcc-rs" }
gcc = "0.3.36"
libc = "0.2"
md5 = "0.1"

View File

@ -10,8 +10,64 @@ system.
## Using rustbuild
When configuring Rust via `./configure`, pass the following to enable building
via this build system:
The rustbuild build system has a primary entry point, a top level `x.py` script:
```
python ./x.py build
```
Note that if you're on Unix you should be able to execute the script directly:
```
./x.py build
```
The script accepts commands, flags, and filters to determine what to do:
* `build` - a general purpose command for compiling code. Alone `build` will
bootstrap the entire compiler, and otherwise arguments passed indicate what to
build. For example:
```
# build the whole compiler
./x.py build
# build the stage1 compier
./x.py build --stage 1
# build stage0 libstd
./x.py build --stage 0 src/libstd
# build a particular crate in stage0
./x.py build --stage 0 src/libtest
```
* `test` - a command for executing unit tests. Like the `build` command this
will execute the entire test suite by default, and otherwise it can be used to
select which test suite is run:
```
# run all unit tests
./x.py test
# execute the run-pass test suite
./x.py test src/test/run-pass
# execute only some tests in the run-pass test suite
./x.py test src/test/run-pass --filter my-filter
# execute tests in the standard library in stage0
./x.py test --stage 0 src/libstd
# execute all doc tests
./x.py test src/doc
```
* `doc` - a command for building documentation. Like above can take arguments
for what to document.
If you're more used to `./configure` and `make`, however, then you can also
configure the build system to use rustbuild instead of the old makefiles:
```
./configure --enable-rustbuild
@ -19,15 +75,7 @@ make
```
Afterwards the `Makefile` which is generated will have a few commands like
`make check`, `make tidy`, etc. For finer-grained control, the
`bootstrap.py` entry point can be used:
```
python src/bootstrap/bootstrap.py
```
This accepts a number of options like `--stage` and `--step` which can configure
what's actually being done.
`make check`, `make tidy`, etc.
## Configuring rustbuild
@ -47,7 +95,7 @@ being invoked manually (via the python script).
The rustbuild build system goes through a few phases to actually build the
compiler. What actually happens when you invoke rustbuild is:
1. The entry point script, `src/bootstrap/bootstrap.py` is run. This script is
1. The entry point script, `x.py` is run. This script is
responsible for downloading the stage0 compiler/Cargo binaries, and it then
compiles the build system itself (this folder). Finally, it then invokes the
actual `bootstrap` binary build system.

View File

@ -399,12 +399,10 @@ def main():
# Run the bootstrap
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
args.append('--src')
args.append(rb.rust_root)
args.append('--build')
args.append(rb.build)
args.extend(sys.argv[1:])
env = os.environ.copy()
env["BUILD"] = rb.build
env["SRC"] = rb.rust_root
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
rb.run(args, env)

View File

@ -13,44 +13,19 @@
//! This file implements the various regression test suites that we execute on
//! our CI.
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::env;
use std::fs;
use std::path::{PathBuf, Path};
use std::process::Command;
use build_helper::output;
use rustc_serialize::json;
use {Build, Compiler, Mode};
use util::{self, dylib_path, dylib_path_var};
const ADB_TEST_DIR: &'static str = "/data/tmp";
#[derive(RustcDecodable)]
struct Output {
packages: Vec<Package>,
resolve: Resolve,
}
#[derive(RustcDecodable)]
struct Package {
id: String,
name: String,
source: Option<String>,
}
#[derive(RustcDecodable)]
struct Resolve {
nodes: Vec<ResolveNode>,
}
#[derive(RustcDecodable)]
struct ResolveNode {
id: String,
dependencies: Vec<String>,
}
/// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
///
/// This tool in `src/tools` will verify the validity of all our links in the
@ -181,7 +156,7 @@ pub fn compiletest(build: &Build,
let llvm_version = output(Command::new(&llvm_config).arg("--version"));
cmd.arg("--llvm-version").arg(llvm_version);
cmd.args(&build.flags.args);
cmd.args(&build.flags.cmd.test_args());
if build.config.verbose || build.flags.verbose {
cmd.arg("--verbose");
@ -282,7 +257,7 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
cmd.arg("--test");
cmd.arg(markdown);
let mut test_args = build.flags.args.join(" ");
let mut test_args = build.flags.cmd.test_args().join(" ");
if build.config.quiet_tests {
test_args.push_str(" --quiet");
}
@ -302,7 +277,8 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
pub fn krate(build: &Build,
compiler: &Compiler,
target: &str,
mode: Mode) {
mode: Mode,
krate: Option<&str>) {
let (name, path, features, root) = match mode {
Mode::Libstd => {
("libstd", "src/rustc/std_shim", build.std_features(), "std_shim")
@ -318,24 +294,6 @@ pub fn krate(build: &Build,
println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
compiler.host, target);
// Run `cargo metadata` to figure out what crates we're testing.
//
// Down below we're going to call `cargo test`, but to test the right set
// of packages we're going to have to know what `-p` arguments to pass it
// to know what crates to test. Here we run `cargo metadata` to learn about
// the dependency graph and what `-p` arguments there are.
let mut cargo = Command::new(&build.cargo);
cargo.arg("metadata")
.arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml"));
let output = output(&mut cargo);
let output: Output = json::decode(&output).unwrap();
let id2pkg = output.packages.iter()
.map(|pkg| (&pkg.id, pkg))
.collect::<HashMap<_, _>>();
let id2deps = output.resolve.nodes.iter()
.map(|node| (&node.id, &node.dependencies))
.collect::<HashMap<_, _>>();
// Build up the base `cargo test` command.
//
// Pass in some standard flags then iterate over the graph we've discovered
@ -346,27 +304,28 @@ pub fn krate(build: &Build,
.arg(build.src.join(path).join("Cargo.toml"))
.arg("--features").arg(features);
let mut visited = HashSet::new();
let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap();
let mut next = vec![&root_pkg.id];
while let Some(id) = next.pop() {
// Skip any packages with sources listed, as these come from crates.io
// and we shouldn't be testing them.
if id2pkg[id].source.is_some() {
continue
match krate {
Some(krate) => {
cargo.arg("-p").arg(krate);
}
None => {
let mut visited = HashSet::new();
let mut next = vec![root];
while let Some(name) = next.pop() {
// Right now jemalloc is our only target-specific crate in the sense
// that it's not present on all platforms. Custom skip it here for now,
// but if we add more this probably wants to get more generalized.
if !id.contains("jemalloc") {
cargo.arg("-p").arg(&id2pkg[id].name);
if !name.contains("jemalloc") {
cargo.arg("-p").arg(name);
}
for dep in id2deps[id] {
for dep in build.crates[name].deps.iter() {
if visited.insert(dep) {
next.push(dep);
}
}
}
}
}
// The tests are going to run with the *target* libraries, so we need to
// ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
@ -389,7 +348,7 @@ pub fn krate(build: &Build,
build.run(cargo.arg("--no-run"));
krate_emscripten(build, compiler, target, mode);
} else {
cargo.args(&build.flags.args);
cargo.args(&build.flags.cmd.test_args());
build.run(&mut cargo);
}
}
@ -421,7 +380,7 @@ fn krate_android(build: &Build,
target = target,
test = test_file_name,
log = log,
args = build.flags.args.join(" "));
args = build.flags.cmd.test_args().join(" "));
let output = output(Command::new("adb").arg("shell").arg(&program));
println!("{}", output);

View File

@ -25,17 +25,17 @@ pub fn clean(build: &Build) {
rm_rf(build, &build.out.join("tmp"));
for host in build.config.host.iter() {
let entries = match build.out.join(host).read_dir() {
Ok(iter) => iter,
Err(_) => continue,
};
let out = build.out.join(host);
rm_rf(build, &out.join("doc"));
for stage in 0..4 {
rm_rf(build, &out.join(format!("stage{}", stage)));
rm_rf(build, &out.join(format!("stage{}-std", stage)));
rm_rf(build, &out.join(format!("stage{}-rustc", stage)));
rm_rf(build, &out.join(format!("stage{}-tools", stage)));
rm_rf(build, &out.join(format!("stage{}-test", stage)));
for entry in entries {
let entry = t!(entry);
if entry.file_name().to_str() == Some("llvm") {
continue
}
t!(fs::remove_dir_all(&entry.path()));
}
}
}

View File

@ -64,8 +64,8 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
}
build.run(&mut cargo);
update_mtime(&libstd_stamp(build, compiler, target));
std_link(build, target, compiler, compiler.host);
update_mtime(&libstd_stamp(build, &compiler, target));
std_link(build, target, compiler.stage, compiler.host);
}
/// Link all libstd rlibs/dylibs into the sysroot location.
@ -74,11 +74,12 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
/// by `compiler` into `host`'s sysroot.
pub fn std_link(build: &Build,
target: &str,
compiler: &Compiler,
stage: u32,
host: &str) {
let compiler = Compiler::new(stage, &build.config.build);
let target_compiler = Compiler::new(compiler.stage, host);
let libdir = build.sysroot_libdir(&target_compiler, target);
let out_dir = build.cargo_out(compiler, Mode::Libstd, target);
let out_dir = build.cargo_out(&compiler, Mode::Libstd, target);
// If we're linking one compiler host's output into another, then we weren't
// called from the `std` method above. In that case we clean out what's
@ -146,7 +147,7 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
.arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
build.run(&mut cargo);
update_mtime(&libtest_stamp(build, compiler, target));
test_link(build, target, compiler, compiler.host);
test_link(build, target, compiler.stage, compiler.host);
}
/// Link all libtest rlibs/dylibs into the sysroot location.
@ -155,11 +156,12 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
/// by `compiler` into `host`'s sysroot.
pub fn test_link(build: &Build,
target: &str,
compiler: &Compiler,
stage: u32,
host: &str) {
let compiler = Compiler::new(stage, &build.config.build);
let target_compiler = Compiler::new(compiler.stage, host);
let libdir = build.sysroot_libdir(&target_compiler, target);
let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
let out_dir = build.cargo_out(&compiler, Mode::Libtest, target);
add_to_sysroot(&out_dir, &libdir);
}
@ -218,7 +220,7 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
}
build.run(&mut cargo);
rustc_link(build, target, compiler, compiler.host);
rustc_link(build, target, compiler.stage, compiler.host);
}
/// Link all librustc rlibs/dylibs into the sysroot location.
@ -227,11 +229,12 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
/// by `compiler` into `host`'s sysroot.
pub fn rustc_link(build: &Build,
target: &str,
compiler: &Compiler,
stage: u32,
host: &str) {
let compiler = Compiler::new(stage, &build.config.build);
let target_compiler = Compiler::new(compiler.stage, host);
let libdir = build.sysroot_libdir(&target_compiler, target);
let out_dir = build.cargo_out(compiler, Mode::Librustc, target);
let out_dir = build.cargo_out(&compiler, Mode::Librustc, target);
add_to_sysroot(&out_dir, &libdir);
}
@ -259,7 +262,10 @@ fn compiler_file(compiler: &Path, file: &str) -> PathBuf {
/// must have been previously produced by the `stage - 1` build.config.build
/// compiler.
pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
assert!(stage > 0, "the stage0 compiler isn't assembled, it's downloaded");
// nothing to do in stage0
if stage == 0 {
return
}
// The compiler that we're assembling
let target_compiler = Compiler::new(stage, host);

View File

@ -19,7 +19,6 @@
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::Path;
use std::process::Command;
use {Build, Compiler, Mode};
@ -30,8 +29,9 @@ use util::{up_to_date, cp_r};
///
/// This will not actually generate any documentation if the documentation has
/// already been generated.
pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str, out: &Path) {
t!(fs::create_dir_all(out));
pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str) {
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let out = out.join(name);
let compiler = Compiler::new(stage, &build.config.build);
@ -57,9 +57,10 @@ pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str, out: &Path)
/// `STAMP` alongw ith providing the various header/footer HTML we've cutomized.
///
/// In the end, this is just a glorified wrapper around rustdoc!
pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) {
pub fn standalone(build: &Build, stage: u32, target: &str) {
println!("Documenting stage{} standalone ({})", stage, target);
t!(fs::create_dir_all(out));
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let compiler = Compiler::new(stage, &build.config.build);
@ -109,7 +110,7 @@ pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) {
.arg("--html-in-header").arg(&favicon)
.arg("--markdown-playground-url")
.arg("https://play.rust-lang.org/")
.arg("-o").arg(out)
.arg("-o").arg(&out)
.arg(&path);
if filename == "reference.md" {
@ -131,9 +132,10 @@ pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) {
///
/// This will generate all documentation for the standard library and its
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn std(build: &Build, stage: u32, target: &str, out: &Path) {
pub fn std(build: &Build, stage: u32, target: &str) {
println!("Documenting stage{} std ({})", stage, target);
t!(fs::create_dir_all(out));
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let compiler = Compiler::new(stage, &build.config.build);
let out_dir = build.stage_out(&compiler, Mode::Libstd)
.join(target).join("doc");
@ -146,16 +148,17 @@ pub fn std(build: &Build, stage: u32, target: &str, out: &Path) {
.arg(build.src.join("src/rustc/std_shim/Cargo.toml"))
.arg("--features").arg(build.std_features());
build.run(&mut cargo);
cp_r(&out_dir, out)
cp_r(&out_dir, &out)
}
/// Compile all libtest documentation.
///
/// This will generate all documentation for libtest and its dependencies. This
/// is largely just a wrapper around `cargo doc`.
pub fn test(build: &Build, stage: u32, target: &str, out: &Path) {
pub fn test(build: &Build, stage: u32, target: &str) {
println!("Documenting stage{} test ({})", stage, target);
t!(fs::create_dir_all(out));
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let compiler = Compiler::new(stage, &build.config.build);
let out_dir = build.stage_out(&compiler, Mode::Libtest)
.join(target).join("doc");
@ -167,16 +170,17 @@ pub fn test(build: &Build, stage: u32, target: &str, out: &Path) {
cargo.arg("--manifest-path")
.arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
build.run(&mut cargo);
cp_r(&out_dir, out)
cp_r(&out_dir, &out)
}
/// Generate all compiler documentation.
///
/// This will generate all documentation for the compiler libraries and their
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn rustc(build: &Build, stage: u32, target: &str, out: &Path) {
pub fn rustc(build: &Build, stage: u32, target: &str) {
println!("Documenting stage{} compiler ({})", stage, target);
t!(fs::create_dir_all(out));
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let compiler = Compiler::new(stage, &build.config.build);
let out_dir = build.stage_out(&compiler, Mode::Librustc)
.join(target).join("doc");
@ -189,14 +193,15 @@ pub fn rustc(build: &Build, stage: u32, target: &str, out: &Path) {
.arg(build.src.join("src/rustc/Cargo.toml"))
.arg("--features").arg(build.rustc_features());
build.run(&mut cargo);
cp_r(&out_dir, out)
cp_r(&out_dir, &out)
}
/// Generates the HTML rendered error-index by running the
/// `error_index_generator` tool.
pub fn error_index(build: &Build, stage: u32, target: &str, out: &Path) {
pub fn error_index(build: &Build, stage: u32, target: &str) {
println!("Documenting stage{} error index ({})", stage, target);
t!(fs::create_dir_all(out));
let out = build.doc_out(target);
t!(fs::create_dir_all(&out));
let compiler = Compiler::new(stage, &build.config.build);
let mut index = build.tool_cmd(&compiler, "error_index_generator");
index.arg("html");

View File

@ -13,30 +13,46 @@
//! This module implements the command-line parsing of the build system which
//! has various flags to configure how it's run.
use std::env;
use std::fs;
use std::path::PathBuf;
use std::process;
use std::slice;
use getopts::Options;
use getopts::{Matches, Options};
use Build;
use config::Config;
use metadata;
use step;
/// Deserialized version of all flags for this compile.
pub struct Flags {
pub verbose: bool,
pub stage: Option<u32>,
pub build: String,
pub host: Filter,
pub target: Filter,
pub step: Vec<String>,
pub host: Vec<String>,
pub target: Vec<String>,
pub config: Option<PathBuf>,
pub src: Option<PathBuf>,
pub jobs: Option<u32>,
pub args: Vec<String>,
pub clean: bool,
pub cmd: Subcommand,
}
pub struct Filter {
values: Vec<String>,
pub enum Subcommand {
Build {
paths: Vec<PathBuf>,
},
Doc {
paths: Vec<PathBuf>,
},
Test {
paths: Vec<PathBuf>,
test_args: Vec<String>,
},
Clean,
Dist {
install: bool,
},
}
impl Flags {
@ -44,29 +60,177 @@ impl Flags {
let mut opts = Options::new();
opts.optflag("v", "verbose", "use verbose output");
opts.optopt("", "config", "TOML configuration file for build", "FILE");
opts.optopt("", "build", "build target of the stage0 compiler", "BUILD");
opts.optmulti("", "host", "host targets to build", "HOST");
opts.reqopt("", "build", "build target of the stage0 compiler", "BUILD");
opts.optmulti("", "target", "targets to build", "TARGET");
opts.optmulti("s", "step", "build step to execute", "STEP");
opts.optmulti("", "target", "target targets to build", "TARGET");
opts.optopt("", "stage", "stage to build", "N");
opts.optopt("", "src", "path to repo root", "DIR");
opts.optopt("", "src", "path to the root of the rust checkout", "DIR");
opts.optopt("j", "jobs", "number of jobs to run in parallel", "JOBS");
opts.optflag("", "clean", "clean output directory");
opts.optflag("h", "help", "print this help message");
let usage = |n| -> ! {
let brief = format!("Usage: rust.py [options]");
print!("{}", opts.usage(&brief));
let usage = |n, opts: &Options| -> ! {
let command = args.get(0).map(|s| &**s);
let brief = format!("Usage: x.py {} [options] [<args>...]",
command.unwrap_or("<command>"));
println!("{}", opts.usage(&brief));
match command {
Some("build") => {
println!("\
Arguments:
This subcommand accepts a number of positional arguments of directories to
the crates and/or artifacts to compile. For example:
./x.py build src/libcore
./x.py build src/libproc_macro
./x.py build src/libstd --stage 1
If no arguments are passed then the complete artifacts for that stage are
also compiled.
./x.py build
./x.py build --stage 1
For a quick build with a usable compile, you can pass:
./x.py build --stage 1 src/libtest
");
}
Some("test") => {
println!("\
Arguments:
This subcommand accepts a number of positional arguments of directories to
tests that should be compiled and run. For example:
./x.py test src/test/run-pass
./x.py test src/test/run-pass/assert-*
./x.py test src/libstd --test-args hash_map
./x.py test src/libstd --stage 0
If no arguments are passed then the complete artifacts for that stage are
compiled and tested.
./x.py test
./x.py test --stage 1
");
}
Some("doc") => {
println!("\
Arguments:
This subcommand accepts a number of positional arguments of directories of
documentation to build. For example:
./x.py doc src/doc/book
./x.py doc src/doc/nomicon
./x.py doc src/libstd
If no arguments are passed then everything is documented:
./x.py doc
./x.py doc --stage 1
");
}
_ => {}
}
if let Some(command) = command {
if command == "build" ||
command == "dist" ||
command == "doc" ||
command == "test" ||
command == "clean" {
println!("Available invocations:");
if args.iter().any(|a| a == "-v") {
let flags = Flags::parse(&["build".to_string()]);
let mut config = Config::default();
config.build = flags.build.clone();
let mut build = Build::new(flags, config);
metadata::build(&mut build);
step::build_rules(&build).print_help(command);
} else {
println!(" ... elided, run `./x.py {} -h -v` to see",
command);
}
println!("");
}
}
println!("\
Subcommands:
build Compile either the compiler or libraries
test Build and run some test suites
doc Build documentation
clean Clean out build directories
dist Build and/or install distribution artifacts
To learn more about a subcommand, run `./x.py <command> -h`
");
process::exit(n);
};
let m = opts.parse(args).unwrap_or_else(|e| {
if args.len() == 0 {
println!("a command must be passed");
usage(1, &opts);
}
let parse = |opts: &Options| {
let m = opts.parse(&args[1..]).unwrap_or_else(|e| {
println!("failed to parse options: {}", e);
usage(1);
usage(1, opts);
});
if m.opt_present("h") {
usage(0);
usage(0, opts);
}
return m
};
let cwd = t!(env::current_dir());
let remaining_as_path = |m: &Matches| {
m.free.iter().map(|p| cwd.join(p)).collect::<Vec<_>>()
};
let m: Matches;
let cmd = match &args[0][..] {
"build" => {
m = parse(&opts);
Subcommand::Build { paths: remaining_as_path(&m) }
}
"doc" => {
m = parse(&opts);
Subcommand::Doc { paths: remaining_as_path(&m) }
}
"test" => {
opts.optmulti("", "test-args", "extra arguments", "ARGS");
m = parse(&opts);
Subcommand::Test {
paths: remaining_as_path(&m),
test_args: m.opt_strs("test-args"),
}
}
"clean" => {
m = parse(&opts);
if m.free.len() > 0 {
println!("clean takes no arguments");
usage(1, &opts);
}
Subcommand::Clean
}
"dist" => {
opts.optflag("", "install", "run installer as well");
m = parse(&opts);
Subcommand::Dist {
install: m.opt_present("install"),
}
}
cmd => {
println!("unknown command: {}", cmd);
usage(1, &opts);
}
};
let cfg_file = m.opt_str("config").map(PathBuf::from).or_else(|| {
if fs::metadata("config.toml").is_ok() {
@ -78,26 +242,27 @@ impl Flags {
Flags {
verbose: m.opt_present("v"),
clean: m.opt_present("clean"),
stage: m.opt_str("stage").map(|j| j.parse().unwrap()),
build: m.opt_str("build").unwrap(),
host: Filter { values: m.opt_strs("host") },
target: Filter { values: m.opt_strs("target") },
step: m.opt_strs("step"),
build: m.opt_str("build").unwrap_or_else(|| {
env::var("BUILD").unwrap()
}),
host: m.opt_strs("host"),
target: m.opt_strs("target"),
config: cfg_file,
src: m.opt_str("src").map(PathBuf::from),
jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()),
args: m.free.clone(),
cmd: cmd,
}
}
}
impl Filter {
pub fn contains(&self, name: &str) -> bool {
self.values.len() == 0 || self.values.iter().any(|s| s == name)
impl Subcommand {
pub fn test_args(&self) -> Vec<&str> {
match *self {
Subcommand::Test { ref test_args, .. } => {
test_args.iter().flat_map(|s| s.split_whitespace()).collect()
}
_ => Vec::new(),
}
pub fn iter(&self) -> slice::Iter<String> {
self.values.iter()
}
}

View File

@ -57,6 +57,7 @@ mod channel;
mod check;
mod clean;
mod compile;
mod metadata;
mod config;
mod dist;
mod doc;
@ -76,7 +77,7 @@ mod job {
}
pub use config::Config;
pub use flags::Flags;
pub use flags::{Flags, Subcommand};
/// A structure representing a Rust compiler.
///
@ -130,6 +131,17 @@ pub struct Build {
// Runtime state filled in later on
cc: HashMap<String, (gcc::Tool, Option<PathBuf>)>,
cxx: HashMap<String, gcc::Tool>,
crates: HashMap<String, Crate>,
}
#[derive(Debug)]
struct Crate {
name: String,
deps: Vec<String>,
path: PathBuf,
doc_step: String,
build_step: String,
test_step: String,
}
/// The various "modes" of invoking Cargo.
@ -162,7 +174,9 @@ impl Build {
/// By default all build output will be placed in the current directory.
pub fn new(flags: Flags, config: Config) -> Build {
let cwd = t!(env::current_dir());
let src = flags.src.clone().unwrap_or(cwd.clone());
let src = flags.src.clone().or_else(|| {
env::var_os("SRC").map(|x| x.into())
}).unwrap_or(cwd.clone());
let out = cwd.join("build");
let stage0_root = out.join(&config.build).join("stage0/bin");
@ -196,6 +210,7 @@ impl Build {
package_vers: String::new(),
cc: HashMap::new(),
cxx: HashMap::new(),
crates: HashMap::new(),
gdb_version: None,
lldb_version: None,
lldb_python_dir: None,
@ -204,13 +219,11 @@ impl Build {
/// Executes the entire build, as configured by the flags and configuration.
pub fn build(&mut self) {
use step::Source::*;
unsafe {
job::setup();
}
if self.flags.clean {
if let Subcommand::Clean = self.flags.cmd {
return clean::clean(self);
}
@ -232,247 +245,10 @@ impl Build {
}
self.verbose("updating submodules");
self.update_submodules();
self.verbose("learning about cargo");
metadata::build(self);
// The main loop of the build system.
//
// The `step::all` function returns a topographically sorted list of all
// steps that need to be executed as part of this build. Each step has a
// corresponding entry in `step.rs` and indicates some unit of work that
// needs to be done as part of the build.
//
// Almost all of these are simple one-liners that shell out to the
// corresponding functionality in the extra modules, where more
// documentation can be found.
let steps = step::all(self);
self.verbose("bootstrap build plan:");
for step in &steps {
self.verbose(&format!("{:?}", step));
}
for target in steps {
let doc_out = self.out.join(&target.target).join("doc");
match target.src {
Llvm { _dummy } => {
native::llvm(self, target.target);
}
TestHelpers { _dummy } => {
native::test_helpers(self, target.target);
}
Libstd { compiler } => {
compile::std(self, target.target, &compiler);
}
Libtest { compiler } => {
compile::test(self, target.target, &compiler);
}
Librustc { compiler } => {
compile::rustc(self, target.target, &compiler);
}
LibstdLink { compiler, host } => {
compile::std_link(self, target.target, &compiler, host);
}
LibtestLink { compiler, host } => {
compile::test_link(self, target.target, &compiler, host);
}
LibrustcLink { compiler, host } => {
compile::rustc_link(self, target.target, &compiler, host);
}
Rustc { stage: 0 } => {
// nothing to do...
}
Rustc { stage } => {
compile::assemble_rustc(self, stage, target.target);
}
ToolLinkchecker { stage } => {
compile::tool(self, stage, target.target, "linkchecker");
}
ToolRustbook { stage } => {
compile::tool(self, stage, target.target, "rustbook");
}
ToolErrorIndex { stage } => {
compile::tool(self, stage, target.target,
"error_index_generator");
}
ToolCargoTest { stage } => {
compile::tool(self, stage, target.target, "cargotest");
}
ToolTidy { stage } => {
compile::tool(self, stage, target.target, "tidy");
}
ToolCompiletest { stage } => {
compile::tool(self, stage, target.target, "compiletest");
}
DocBook { stage } => {
doc::rustbook(self, stage, target.target, "book", &doc_out);
}
DocNomicon { stage } => {
doc::rustbook(self, stage, target.target, "nomicon",
&doc_out);
}
DocStandalone { stage } => {
doc::standalone(self, stage, target.target, &doc_out);
}
DocStd { stage } => {
doc::std(self, stage, target.target, &doc_out);
}
DocTest { stage } => {
doc::test(self, stage, target.target, &doc_out);
}
DocRustc { stage } => {
doc::rustc(self, stage, target.target, &doc_out);
}
DocErrorIndex { stage } => {
doc::error_index(self, stage, target.target, &doc_out);
}
CheckLinkcheck { stage } => {
check::linkcheck(self, stage, target.target);
}
CheckCargoTest { stage } => {
check::cargotest(self, stage, target.target);
}
CheckTidy { stage } => {
check::tidy(self, stage, target.target);
}
CheckRPass { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-pass", "run-pass");
}
CheckRPassFull { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-pass", "run-pass-fulldeps");
}
CheckCFail { compiler } => {
check::compiletest(self, &compiler, target.target,
"compile-fail", "compile-fail");
}
CheckCFailFull { compiler } => {
check::compiletest(self, &compiler, target.target,
"compile-fail", "compile-fail-fulldeps")
}
CheckPFail { compiler } => {
check::compiletest(self, &compiler, target.target,
"parse-fail", "parse-fail");
}
CheckRFail { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-fail", "run-fail");
}
CheckRFailFull { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-fail", "run-fail-fulldeps");
}
CheckPretty { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "pretty");
}
CheckPrettyRPass { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "run-pass");
}
CheckPrettyRPassFull { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "run-pass-fulldeps");
}
CheckPrettyRFail { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "run-fail");
}
CheckPrettyRFailFull { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "run-fail-fulldeps");
}
CheckPrettyRPassValgrind { compiler } => {
check::compiletest(self, &compiler, target.target,
"pretty", "run-pass-valgrind");
}
CheckMirOpt { compiler } => {
check::compiletest(self, &compiler, target.target,
"mir-opt", "mir-opt");
}
CheckCodegen { compiler } => {
if self.config.codegen_tests {
check::compiletest(self, &compiler, target.target,
"codegen", "codegen");
}
}
CheckCodegenUnits { compiler } => {
check::compiletest(self, &compiler, target.target,
"codegen-units", "codegen-units");
}
CheckIncremental { compiler } => {
check::compiletest(self, &compiler, target.target,
"incremental", "incremental");
}
CheckUi { compiler } => {
check::compiletest(self, &compiler, target.target,
"ui", "ui");
}
CheckDebuginfo { compiler } => {
if target.target.contains("msvc") {
// nothing to do
} else if target.target.contains("apple") {
check::compiletest(self, &compiler, target.target,
"debuginfo-lldb", "debuginfo");
} else {
check::compiletest(self, &compiler, target.target,
"debuginfo-gdb", "debuginfo");
}
}
CheckRustdoc { compiler } => {
check::compiletest(self, &compiler, target.target,
"rustdoc", "rustdoc");
}
CheckRPassValgrind { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-pass-valgrind", "run-pass-valgrind");
}
CheckDocs { compiler } => {
check::docs(self, &compiler);
}
CheckErrorIndex { compiler } => {
check::error_index(self, &compiler);
}
CheckRMake { compiler } => {
check::compiletest(self, &compiler, target.target,
"run-make", "run-make")
}
CheckCrateStd { compiler } => {
check::krate(self, &compiler, target.target, Mode::Libstd)
}
CheckCrateTest { compiler } => {
check::krate(self, &compiler, target.target, Mode::Libtest)
}
CheckCrateRustc { compiler } => {
check::krate(self, &compiler, target.target, Mode::Librustc)
}
DistDocs { stage } => dist::docs(self, stage, target.target),
DistMingw { _dummy } => dist::mingw(self, target.target),
DistRustc { stage } => dist::rustc(self, stage, target.target),
DistStd { compiler } => dist::std(self, &compiler, target.target),
DistSrc { _dummy } => dist::rust_src(self),
Install { stage } => install::install(self, stage, target.target),
DebuggerScripts { stage } => {
let compiler = Compiler::new(stage, target.target);
dist::debugger_scripts(self,
&self.sysroot(&compiler),
target.target);
}
AndroidCopyLibs { compiler } => {
check::android_copy_libs(self, &compiler, target.target);
}
// pseudo-steps
Dist { .. } |
Doc { .. } |
CheckTarget { .. } |
Check { .. } => {}
}
}
step::run(self);
}
/// Updates all git submodules that we have.
@ -812,6 +588,11 @@ impl Build {
self.out.join(target).join("llvm")
}
/// Output directory for all documentation for a target
fn doc_out(&self, target: &str) -> PathBuf {
self.out.join(target).join("doc")
}
/// Returns true if no custom `llvm-config` is set for the specified target.
///
/// If no custom `llvm-config` was specified then Rust's llvm will be used.

95
src/bootstrap/metadata.rs Normal file
View File

@ -0,0 +1,95 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::collections::HashMap;
use std::process::Command;
use std::path::PathBuf;
use build_helper::output;
use rustc_serialize::json;
use {Build, Crate};
#[derive(RustcDecodable)]
struct Output {
packages: Vec<Package>,
resolve: Resolve,
}
#[derive(RustcDecodable)]
struct Package {
id: String,
name: String,
source: Option<String>,
manifest_path: String,
}
#[derive(RustcDecodable)]
struct Resolve {
nodes: Vec<ResolveNode>,
}
#[derive(RustcDecodable)]
struct ResolveNode {
id: String,
dependencies: Vec<String>,
}
pub fn build(build: &mut Build) {
build_krate(build, "src/rustc/std_shim");
build_krate(build, "src/rustc/test_shim");
build_krate(build, "src/rustc");
}
fn build_krate(build: &mut Build, krate: &str) {
// Run `cargo metadata` to figure out what crates we're testing.
//
// Down below we're going to call `cargo test`, but to test the right set
// of packages we're going to have to know what `-p` arguments to pass it
// to know what crates to test. Here we run `cargo metadata` to learn about
// the dependency graph and what `-p` arguments there are.
let mut cargo = Command::new(&build.cargo);
cargo.arg("metadata")
.arg("--manifest-path").arg(build.src.join(krate).join("Cargo.toml"));
let output = output(&mut cargo);
let output: Output = json::decode(&output).unwrap();
let mut id2name = HashMap::new();
for package in output.packages {
if package.source.is_none() {
id2name.insert(package.id, package.name.clone());
let mut path = PathBuf::from(package.manifest_path);
path.pop();
build.crates.insert(package.name.clone(), Crate {
build_step: format!("build-crate-{}", package.name),
doc_step: format!("doc-crate-{}", package.name),
test_step: format!("test-crate-{}", package.name),
name: package.name,
deps: Vec::new(),
path: path,
});
}
}
for node in output.resolve.nodes {
let name = match id2name.get(&node.id) {
Some(name) => name,
None => continue,
};
let krate = build.crates.get_mut(name).unwrap();
for dep in node.dependencies.iter() {
let dep = match id2name.get(dep) {
Some(dep) => dep,
None => continue,
};
krate.deps.push(dep.clone());
}
}
}

View File

@ -17,47 +17,46 @@ else
BOOTSTRAP_ARGS :=
endif
BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py $(BOOTSTRAP_ARGS)
BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py
all:
$(Q)$(BOOTSTRAP)
$(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS)
$(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS)
# Dont use $(Q) here, always show how to invoke the bootstrap script directly
help:
$(BOOTSTRAP) --help
clean:
$(Q)$(BOOTSTRAP) --clean
$(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS)
rustc-stage1:
$(Q)$(BOOTSTRAP) --step libtest --stage 1
$(Q)$(BOOTSTRAP) build --stage 1 src/libtest $(BOOTSTRAP_ARGS)
rustc-stage2:
$(Q)$(BOOTSTRAP) --step libtest --stage 2
$(Q)$(BOOTSTRAP) build --stage 2 src/libtest $(BOOTSTRAP_ARGS)
docs: doc
doc:
$(Q)$(BOOTSTRAP) --step doc
style:
$(Q)$(BOOTSTRAP) --step doc-style
$(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS)
nomicon:
$(Q)$(BOOTSTRAP) --step doc-nomicon
$(Q)$(BOOTSTRAP) doc src/doc/nomicon $(BOOTSTRAP_ARGS)
book:
$(Q)$(BOOTSTRAP) --step doc-book
$(Q)$(BOOTSTRAP) doc src/doc/book $(BOOTSTRAP_ARGS)
standalone-docs:
$(Q)$(BOOTSTRAP) --step doc-standalone
$(Q)$(BOOTSTRAP) doc src/doc $(BOOTSTRAP_ARGS)
check:
$(Q)$(BOOTSTRAP) --step check
$(Q)$(BOOTSTRAP) test $(BOOTSTRAP_ARGS)
check-cargotest:
$(Q)$(BOOTSTRAP) --step check-cargotest
$(Q)$(BOOTSTRAP) test src/tools/cargotest $(BOOTSTRAP_ARGS)
dist:
$(Q)$(BOOTSTRAP) --step dist
$(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
install:
ifeq (root user, $(USER) $(patsubst %,user,$(SUDO_USER)))
$(Q)echo "'sudo make install' is not supported currently."
else
$(Q)$(BOOTSTRAP) --step install
$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
endif
tidy:
$(Q)$(BOOTSTRAP) --step check-tidy --stage 0
$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
.PHONY: dist

File diff suppressed because it is too large Load Diff

View File

@ -57,8 +57,7 @@ pub fn cp_r(src: &Path, dst: &Path) {
let name = path.file_name().unwrap();
let dst = dst.join(name);
if t!(f.file_type()).is_dir() {
let _ = fs::remove_dir_all(&dst);
t!(fs::create_dir(&dst));
t!(fs::create_dir_all(&dst));
cp_r(&path, &dst);
} else {
let _ = fs::remove_file(&dst);

19
x.py Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env python
# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import sys
import os
dir = os.path.dirname(__file__)
sys.path.append(os.path.abspath(os.path.join(dir, "src", "bootstrap")))
import bootstrap
bootstrap.main()