Auto merge of #31884 - alexcrichton:no-bootstrap-from-existing, r=brson

These commits add support to the rustbuild build system to compile non-build-system compilers. For example this will allow creating an ARM compiler on a x86 system. The high level way this works is:

* The only compiler ever run is the build target compiler. No other compiler is assumed to be runnable.
* As a result, all output artifacts come from the build compiler.
* The libs for the stageN cross-compiled compiler will be linked into place after the build target builds them.

This will break the assumption that a compiler never links to anything it didn't itself produce, but it retains the assumption that a compiler only ever links to anything built in the same stage. I believe this means that the stage1 cross-compiled compilers will still be usable.

I tested this by creating an `arm-unknown-linux-gnueabihf` compiler. So far the linking ended up all working OK (including LLVM being cross compiled), but I haven't been able to run it yet (in QEMU for a raspberry pi). I think that's because my system linker is messed up or something like that (some newer option is assumed when it's not actually there).

Overall, though, this means that rustbuild can compile an arm-unknown-linux-gnueabihf compiler. Ideally we could even start shipping nightlies based on this at some point!
This commit is contained in:
bors 2016-02-29 22:30:44 +00:00
commit 5a0308abad
10 changed files with 216 additions and 101 deletions

View File

@ -3,57 +3,42 @@ name = "bootstrap"
version = "0.0.0"
dependencies = [
"build_helper 0.1.0",
"cmake 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"cmake 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)",
"getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "advapi32-sys"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "build_helper"
version = "0.1.0"
[[package]]
name = "cmake"
version = "0.1.10"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gcc 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "filetime"
version = "0.1.8"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gcc"
version = "0.3.19"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"advapi32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "getopts"
@ -71,30 +56,28 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.2"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "num_cpus"
version = "0.2.9"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rustc-serialize"
version = "0.3.16"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "toml"
version = "0.1.23"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]

View File

@ -73,6 +73,7 @@ class RustBuild:
if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
shutil.rmtree(self.bin_root())
filename = "rust-std-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
tarball = os.path.join(rustc_cache, filename)

View File

@ -58,6 +58,30 @@ pub fn std<'a>(build: &'a Build, stage: u32, target: &str,
}
build.run(&mut cargo);
std_link(build, stage, target, compiler, host);
}
/// Link all libstd rlibs/dylibs into the sysroot location.
///
/// Links those artifacts generated in the given `stage` for `target` produced
/// by `compiler` into `host`'s sysroot.
pub fn std_link(build: &Build,
stage: u32,
target: &str,
compiler: &Compiler,
host: &str) {
let libdir = build.sysroot_libdir(stage, host, target);
let out_dir = build.cargo_out(stage, compiler.host, true, target);
// If we're linking one compiler host's output into another, then we weren't
// called from the `std` method above. In that case we clean out what's
// already there and then also link compiler-rt into place.
if host != compiler.host {
let _ = fs::remove_dir_all(&libdir);
t!(fs::create_dir_all(&libdir));
t!(fs::hard_link(&build.compiler_rt_built.borrow()[target],
libdir.join(staticlib("compiler-rt", target))));
}
add_to_sysroot(&out_dir, &libdir);
}
@ -99,7 +123,6 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
host, target);
let out_dir = build.cargo_out(stage, &host, false, target);
let rustc = out_dir.join(exe("rustc", target));
build.clear_if_dirty(&out_dir, &libstd_shim(build, stage, &host, target));
let mut cargo = build.cargo(stage, compiler, false, target, "build");
@ -131,10 +154,13 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
if !build.unstable_features {
cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1");
}
if let Some(config) = build.config.target_config.get(target) {
if let Some(ref s) = config.llvm_config {
cargo.env("LLVM_CONFIG", s);
}
let target_config = build.config.target_config.get(target);
if let Some(ref s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
cargo.env("LLVM_CONFIG", s);
} else {
let llvm_config = build.llvm_out(&build.config.build).join("bin")
.join(exe("llvm-config", target));
cargo.env("LLVM_CONFIG", llvm_config);
}
if build.config.llvm_static_stdcpp {
cargo.env("LLVM_STATIC_STDCPP",
@ -148,12 +174,21 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
}
build.run(&mut cargo);
let sysroot_libdir = build.sysroot_libdir(stage, host, target);
add_to_sysroot(&out_dir, &sysroot_libdir);
rustc_link(build, stage, target, compiler, compiler.host);
}
if host == target {
assemble_compiler(build, stage, target, &rustc);
}
/// Link all librustc rlibs/dylibs into the sysroot location.
///
/// Links those artifacts generated in the given `stage` for `target` produced
/// by `compiler` into `host`'s sysroot.
pub fn rustc_link(build: &Build,
stage: u32,
target: &str,
compiler: &Compiler,
host: &str) {
let libdir = build.sysroot_libdir(stage, host, target);
let out_dir = build.cargo_out(stage, compiler.host, false, target);
add_to_sysroot(&out_dir, &libdir);
}
/// Cargo's output path for the standard library in a given stage, compiled
@ -169,21 +204,21 @@ fn compiler_file(compiler: &Path, file: &str) -> String {
/// Prepare a new compiler from the artifacts in `stage`
///
/// This will link the compiler built by `host` during the stage
/// specified to the sysroot location for `host` to be the official
/// `stage + 1` compiler for that host. This means that the `rustc` binary
/// itself will be linked into place along with all supporting dynamic
/// libraries.
fn assemble_compiler(build: &Build, stage: u32, host: &str, rustc: &Path) {
/// This will assemble a compiler in `build/$host/stage$stage`. The compiler
/// must have been previously produced by the `stage - 1` build.config.build
/// compiler.
pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
assert!(stage > 0, "the stage0 compiler isn't assembled, it's downloaded");
// Clear out old files
let sysroot = build.sysroot(stage + 1, host);
let sysroot = build.sysroot(stage, host);
let _ = fs::remove_dir_all(&sysroot);
t!(fs::create_dir_all(&sysroot));
// Link in all dylibs to the libdir
let sysroot_libdir = sysroot.join(libdir(host));
t!(fs::create_dir_all(&sysroot_libdir));
let src_libdir = build.sysroot_libdir(stage, host, host);
let src_libdir = build.sysroot_libdir(stage - 1, &build.config.build, host);
for f in t!(fs::read_dir(&src_libdir)).map(|f| t!(f)) {
let filename = f.file_name().into_string().unwrap();
if is_dylib(&filename) {
@ -191,17 +226,20 @@ fn assemble_compiler(build: &Build, stage: u32, host: &str, rustc: &Path) {
}
}
let out_dir = build.cargo_out(stage - 1, &build.config.build, false, host);
// Link the compiler binary itself into place
let rustc = out_dir.join(exe("rustc", host));
let bindir = sysroot.join("bin");
t!(fs::create_dir_all(&bindir));
let compiler = build.compiler_path(&Compiler::new(stage + 1, host));
let compiler = build.compiler_path(&Compiler::new(stage, host));
let _ = fs::remove_file(&compiler);
t!(fs::hard_link(rustc, compiler));
// See if rustdoc exists to link it into place
let exe = exe("rustdoc", host);
let rustdoc_src = rustc.parent().unwrap().join(&exe);
let rustdoc_dst = bindir.join(exe);
let rustdoc = exe("rustdoc", host);
let rustdoc_src = out_dir.join(&rustdoc);
let rustdoc_dst = bindir.join(&rustdoc);
if fs::metadata(&rustdoc_src).is_ok() {
let _ = fs::remove_file(&rustdoc_dst);
t!(fs::hard_link(&rustdoc_src, &rustdoc_dst));

View File

@ -39,6 +39,14 @@ mod sanity;
mod step;
mod util;
#[cfg(windows)]
mod job;
#[cfg(not(windows))]
mod job {
pub unsafe fn setup() {}
}
pub use build::config::Config;
pub use build::flags::Flags;
@ -114,14 +122,9 @@ impl Build {
pub fn build(&mut self) {
use build::step::Source::*;
// see comments in job.rs for what's going on here
#[cfg(windows)]
fn setup_job() {
mod job;
unsafe { job::setup() }
unsafe {
job::setup();
}
#[cfg(not(windows))] fn setup_job() {}
setup_job();
if self.flags.clean {
return clean::clean(self);
@ -146,8 +149,19 @@ impl Build {
Librustc { stage, compiler } => {
compile::rustc(self, stage, target.target, &compiler);
}
LibstdLink { stage, compiler, host } => {
compile::std_link(self, stage, target.target,
&compiler, host);
}
LibrustcLink { stage, compiler, host } => {
compile::rustc_link(self, stage, target.target,
&compiler, host);
}
Rustc { stage: 0 } => {
// nothing to do...
}
Rustc { stage } => {
println!("ok, rustc stage{} in {}", stage, target.target);
compile::assemble_rustc(self, stage, target.target);
}
}
}
@ -425,21 +439,7 @@ impl Build {
}
fn rustc_flags(&self, target: &str) -> Vec<String> {
let mut base = match target {
"arm-unknown-linux-gnueabihf" => {
vec!["-Ctarget-feature=+v6,+vfp2".to_string()]
}
"mips-unknown-linux-gnu" => {
vec!["-Ctarget-cpu=mips32r2".to_string(),
"-Ctarget-feature=+mips32r2".to_string(),
"-Csoft-float".to_string()]
}
"mipsel-unknown-linux-gnu" => {
vec!["-Ctarget-cpu=mips32".to_string(),
"-Ctarget-feature=+mips32".to_string()]
}
_ => Vec::new(),
};
let mut base = Vec::new();
if target != self.config.build && !target.contains("msvc") {
base.push(format!("-Clinker={}", self.cc(target).display()));
}

View File

@ -115,6 +115,11 @@ pub fn compiler_rt(build: &Build, target: &str) {
let mode = if build.config.rust_optimize {"Release"} else {"Debug"};
let (dir, build_target, libname) = if target.contains("linux") {
let os = if target.contains("android") {"-android"} else {""};
let arch = if arch.starts_with("arm") && target.contains("eabihf") {
"armhf"
} else {
arch
};
let target = format!("clang_rt.builtins-{}{}", arch, os);
("linux".to_string(), target.clone(), target)
} else if target.contains("darwin") {
@ -151,7 +156,10 @@ pub fn compiler_rt(build: &Build, target: &str) {
.define("COMPILER_RT_DEFAULT_TARGET_TRIPLE", target)
.define("COMPILER_RT_BUILD_SANITIZERS", "OFF")
.define("COMPILER_RT_BUILD_EMUTLS", "OFF")
// inform about c/c++ compilers, the c++ compiler isn't actually used but
// it's needed to get the initial configure to work on all platforms.
.define("CMAKE_C_COMPILER", build.cc(target))
.define("CMAKE_CXX_COMPILER", build.cc(target))
.build_target(&build_target);
cfg.build();
}

View File

@ -21,9 +21,36 @@ pub struct Step<'a> {
macro_rules! targets {
($m:ident) => {
$m! {
// Step representing building the stageN compiler. This is just the
// compiler executable itself, not any of the support libraries
(rustc, Rustc { stage: u32 }),
// Steps for the two main cargo builds, one for the standard library
// and one for the compiler itself. These are parameterized over the
// stage output they're going to be placed in along with the
// compiler which is producing the copy of libstd or librustc
(libstd, Libstd { stage: u32, compiler: Compiler<'a> }),
(librustc, Librustc { stage: u32, compiler: Compiler<'a> }),
// Links the standard library/librustc produced by the compiler
// provided into the host's directory also provided.
(libstd_link, LibstdLink {
stage: u32,
compiler: Compiler<'a>,
host: &'a str
}),
(librustc_link, LibrustcLink {
stage: u32,
compiler: Compiler<'a>,
host: &'a str
}),
// Steps for long-running native builds. Ideally these wouldn't
// actually exist and would be part of build scripts, but for now
// these are here.
//
// There aren't really any parameters to this, but empty structs
// with braces are unstable so we just pick something that works.
(llvm, Llvm { _dummy: () }),
(compiler_rt, CompilerRt { _dummy: () }),
}
@ -93,13 +120,25 @@ fn top_level(build: &Build) -> Vec<Step> {
continue
}
let host = t.target(host);
targets.push(host.librustc(stage, host.compiler(stage)));
if host.target == build.config.build {
targets.push(host.librustc(stage, host.compiler(stage)));
} else {
targets.push(host.librustc_link(stage, t.compiler(stage),
host.target));
}
for target in build.config.target.iter() {
if !build.flags.target.contains(target) {
continue
}
targets.push(host.target(target)
.libstd(stage, host.compiler(stage)));
if host.target == build.config.build {
targets.push(host.target(target)
.libstd(stage, host.compiler(stage)));
} else {
targets.push(host.target(target)
.libstd_link(stage, t.compiler(stage),
host.target));
}
}
}
}
@ -114,10 +153,14 @@ fn add_steps<'a>(build: &'a Build,
target: &Step<'a>,
targets: &mut Vec<Step<'a>>) {
for step in build.flags.step.iter() {
let compiler = host.compiler(stage);
let compiler = host.target(&build.config.build).compiler(stage);
match &step[..] {
"libstd" => targets.push(target.libstd(stage, compiler)),
"librustc" => targets.push(target.libstd(stage, compiler)),
"librustc" => targets.push(target.librustc(stage, compiler)),
"libstd-link" => targets.push(target.libstd_link(stage, compiler,
host.target)),
"librustc-link" => targets.push(target.librustc_link(stage, compiler,
host.target)),
"rustc" => targets.push(host.rustc(stage)),
"llvm" => targets.push(target.llvm(())),
"compiler-rt" => targets.push(target.compiler_rt(())),
@ -151,15 +194,11 @@ impl<'a> Step<'a> {
pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
match self.src {
Source::Rustc { stage: 0 } => {
if self.target == build.config.build {
Vec::new()
} else {
let compiler = Compiler::new(0, &build.config.build);
vec![self.librustc(0, compiler)]
}
Vec::new()
}
Source::Rustc { stage } => {
vec![self.librustc(stage - 1, self.compiler(stage - 1))]
let compiler = Compiler::new(stage - 1, &build.config.build);
vec![self.librustc(stage - 1, compiler)]
}
Source::Librustc { stage, compiler } => {
vec![self.libstd(stage, compiler), self.llvm(())]
@ -168,6 +207,14 @@ impl<'a> Step<'a> {
vec![self.compiler_rt(()),
self.rustc(compiler.stage).target(compiler.host)]
}
Source::LibrustcLink { stage, compiler, host } => {
vec![self.librustc(stage, compiler),
self.libstd_link(stage, compiler, host)]
}
Source::LibstdLink { stage, compiler, host } => {
vec![self.libstd(stage, compiler),
self.target(host).rustc(stage)]
}
Source::CompilerRt { _dummy } => {
vec![self.llvm(()).target(&build.config.build)]
}

View File

@ -61,6 +61,9 @@ fn main() {
root.push("/lib");
cmd.arg("-L").arg(&root);
}
if let Ok(s) = env::var("RUSTC_FLAGS") {
cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
}
}
// Set various options from config.toml to configure how we're building
@ -79,9 +82,6 @@ fn main() {
if let Ok(s) = env::var("RUSTC_CODEGEN_UNITS") {
cmd.arg("-C").arg(format!("codegen-units={}", s));
}
if let Ok(s) = env::var("RUSTC_FLAGS") {
cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
}
// Actually run the compiler!
std::process::exit(match cmd.status() {

View File

@ -38,6 +38,25 @@ fn main() {
println!("cargo:rerun-if-changed={}", llvm_config.display());
// Test whether we're cross-compiling LLVM. This is a pretty rare case
// currently where we're producing an LLVM for a different platform than
// what this build script is currently running on.
//
// In that case, there's no guarantee that we can actually run the target,
// so the build system works around this by giving us the LLVM_CONFIG for
// the host platform. This only really works if the host LLVM and target
// LLVM are compiled the same way, but for us that's typically the case.
//
// We detect this cross compiling situation by asking llvm-config what it's
// host-target is. If that's not the TARGET, then we're cross compiling.
// This generally just means that we can't trust all the output of
// llvm-config becaues it might be targeted for the host rather than the
// target.
let target = env::var("TARGET").unwrap();
let host = output(Command::new(&llvm_config).arg("--host-target"));
let host = host.trim();
let is_crossed = target != host;
let optional_components = ["x86", "arm", "aarch64", "mips", "powerpc",
"pnacl"];
@ -69,6 +88,10 @@ fn main() {
let cxxflags = output(&mut cmd);
let mut cfg = gcc::Config::new();
for flag in cxxflags.split_whitespace() {
// Ignore flags like `-m64` when we're doing a cross build
if is_crossed && flag.starts_with("-m") {
continue
}
cfg.flag(flag);
}
cfg.file("../rustllvm/ExecutionEngineWrapper.cpp")
@ -79,9 +102,16 @@ fn main() {
.cpp_link_stdlib(None) // we handle this below
.compile("librustllvm.a");
// Link in all LLVM libraries
// Link in all LLVM libraries, if we're uwring the "wrong" llvm-config then
// we don't pick up system libs because unfortunately they're for the host
// of llvm-config, not the target that we're attempting to link.
let mut cmd = Command::new(&llvm_config);
cmd.arg("--libs").arg("--system-libs").args(&components[..]);
cmd.arg("--libs");
if !is_crossed {
cmd.arg("--system-libs");
}
cmd.args(&components[..]);
for lib in output(&mut cmd).split_whitespace() {
let name = if lib.starts_with("-l") {
&lib[2..]
@ -105,10 +135,20 @@ fn main() {
}
// LLVM ldflags
//
// If we're a cross-compile of LLVM then unfortunately we can't trust these
// ldflags (largely where all the LLVM libs are located). Currently just
// hack around this by replacing the host triple with the target and pray
// that those -L directories are the same!
let mut cmd = Command::new(&llvm_config);
cmd.arg("--ldflags");
for lib in output(&mut cmd).split_whitespace() {
if lib.starts_with("-l") {
if is_crossed {
if lib.starts_with("-L") {
println!("cargo:rustc-link-search=native={}",
lib[2..].replace(&host, &target));
}
} else if lib.starts_with("-l") {
println!("cargo:rustc-link-lib={}", &lib[2..]);
} else if lib.starts_with("-L") {
println!("cargo:rustc-link-search=native={}", &lib[2..]);

View File

@ -1,2 +1,2 @@
rustc: 2015-12-19
rustc: 2016-02-17
cargo: 2016-01-21

View File

@ -18,9 +18,7 @@ dependencies = [
name = "alloc"
version = "0.0.0"
dependencies = [
"alloc_system 0.0.0",
"core 0.0.0",
"libc 0.0.0",
]
[[package]]