From 607e85110ef9c79ce5a52286bb69d385471bc675 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sun, 14 Jun 2020 15:57:21 -0700 Subject: [PATCH] Switch bootstrap metadata to --no-deps. This should run much faster. There are also some drive-by cleanups here to try to simplify things. Also, the paths for in-tree crates are now displayed as relative in `x.py test -h -v`. --- src/bootstrap/builder.rs | 6 +++-- src/bootstrap/doc.rs | 20 ++------------ src/bootstrap/lib.rs | 25 +++++++++++------- src/bootstrap/metadata.rs | 55 +++++++++------------------------------ src/bootstrap/test.rs | 8 ++---- 5 files changed, 37 insertions(+), 77 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index ffdd8485181..345af600c2a 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -255,7 +255,8 @@ impl<'a> ShouldRun<'a> { pub fn all_krates(mut self, name: &str) -> Self { let mut set = BTreeSet::new(); for krate in self.builder.in_tree_crates(name) { - set.insert(PathBuf::from(&krate.path)); + let path = krate.local_path(self.builder); + set.insert(path); } self.paths.insert(PathSet::Set(set)); self @@ -263,7 +264,8 @@ impl<'a> ShouldRun<'a> { pub fn krate(mut self, name: &str) -> Self { for krate in self.builder.in_tree_crates(name) { - self.paths.insert(PathSet::one(&krate.path)); + let path = krate.local_path(self.builder); + self.paths.insert(PathSet::one(path)); } self } diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 5c01c5e852c..6d7fb7acfcb 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -548,8 +548,8 @@ impl Step for Rustc { // Find dependencies for top level crates. let mut compiler_crates = HashSet::new(); for root_crate in &["rustc_driver", "rustc_codegen_llvm", "rustc_codegen_ssa"] { - let interned_root_crate = INTERNER.intern_str(root_crate); - find_compiler_crates(builder, &interned_root_crate, &mut compiler_crates); + compiler_crates + .extend(builder.in_tree_crates(root_crate).into_iter().map(|krate| krate.name)); } for krate in &compiler_crates { @@ -564,22 +564,6 @@ impl Step for Rustc { } } -fn find_compiler_crates( - builder: &Builder<'_>, - name: &Interned, - crates: &mut HashSet>, -) { - // Add current crate. - crates.insert(*name); - - // Look for dependencies. - for dep in builder.crates.get(name).unwrap().deps.iter() { - if builder.crates.get(dep).unwrap().is_local(builder) { - find_compiler_crates(builder, dep, crates); - } - } -} - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Rustdoc { stage: u32, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a125b49fc01..9d3830da390 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -270,12 +270,7 @@ struct Crate { } impl Crate { - fn is_local(&self, build: &Build) -> bool { - self.path.starts_with(&build.config.src) - } - fn local_path(&self, build: &Build) -> PathBuf { - assert!(self.is_local(build)); self.path.strip_prefix(&build.config.src).unwrap().into() } } @@ -1079,17 +1074,29 @@ impl Build { } } + /// Returns a Vec of all the dependencies of the given root crate, + /// including transitive dependencies and the root itself. Only includes + /// "local" crates (those in the local source tree, not from a registry). fn in_tree_crates(&self, root: &str) -> Vec<&Crate> { let mut ret = Vec::new(); let mut list = vec![INTERNER.intern_str(root)]; let mut visited = HashSet::new(); while let Some(krate) = list.pop() { let krate = &self.crates[&krate]; - if krate.is_local(self) { - ret.push(krate); - } + ret.push(krate); for dep in &krate.deps { - if visited.insert(dep) && dep != "build_helper" { + // Don't include optional deps if their features are not + // enabled. Ideally this would be computed from `cargo + // metadata --features …`, but that is somewhat slow. Just + // skip `build_helper` since there aren't any operations we + // want to perform on it. In the future, we may want to + // consider just filtering all build and dev dependencies in + // metadata::build. + if visited.insert(dep) + && dep != "build_helper" + && (dep != "profiler_builtins" || self.config.profiler) + && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + { list.push(*dep); } } diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs index 185f0ddb831..a38391c7b88 100644 --- a/src/bootstrap/metadata.rs +++ b/src/bootstrap/metadata.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; -use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; @@ -12,7 +10,6 @@ use crate::{Build, Crate}; #[derive(Deserialize)] struct Output { packages: Vec, - resolve: Resolve, } #[derive(Deserialize)] @@ -21,38 +18,25 @@ struct Package { name: String, source: Option, manifest_path: String, + dependencies: Vec, } #[derive(Deserialize)] -struct Resolve { - nodes: Vec, -} - -#[derive(Deserialize)] -struct ResolveNode { - id: String, - dependencies: Vec, +struct Dependency { + name: String, + source: Option, } pub fn build(build: &mut Build) { // Run `cargo metadata` to figure out what crates we're testing. - let features: Vec<_> = build - .std_features() - .split_whitespace() - .map(|f| format!("test/{}", f)) - .chain(build.rustc_features().split_whitespace().map(|f| format!("rustc-main/{}", f))) - .collect(); let mut cargo = Command::new(&build.initial_cargo); cargo .arg("metadata") .arg("--format-version") .arg("1") - .arg("--features") - .arg(features.join(",")) - .arg("-Zpackage-features") + .arg("--no-deps") .arg("--manifest-path") - .arg(build.src.join("Cargo.toml")) - .env("RUSTC_BOOTSTRAP", "1"); + .arg(build.src.join("Cargo.toml")); let output = output(&mut cargo); let output: Output = serde_json::from_str(&output).unwrap(); for package in output.packages { @@ -60,26 +44,13 @@ pub fn build(build: &mut Build) { let name = INTERNER.intern_string(package.name); let mut path = PathBuf::from(package.manifest_path); path.pop(); - build.crates.insert(name, Crate { name, id: package.id, deps: HashSet::new(), path }); - } - } - - let id2name: HashMap<_, _> = - build.crates.iter().map(|(name, krate)| (krate.id.clone(), name.clone())).collect(); - - for node in output.resolve.nodes { - let name = match id2name.get(&node.id) { - Some(name) => name, - None => continue, - }; - - let krate = build.crates.get_mut(name).unwrap(); - for dep in node.dependencies.iter() { - let dep = match id2name.get(dep) { - Some(dep) => dep, - None => continue, - }; - krate.deps.insert(*dep); + let deps = package + .dependencies + .into_iter() + .filter(|dep| dep.source.is_none()) + .map(|dep| INTERNER.intern_string(dep.name)) + .collect(); + build.crates.insert(name, Crate { name, id: package.id, deps, path }); } } } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 8659acf1cc5..c1d0316920b 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1651,12 +1651,8 @@ impl Step for Crate { type Output = (); const DEFAULT: bool = true; - fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - for krate in run.builder.in_tree_crates("test") { - run = run.path(krate.local_path(&builder).to_str().unwrap()); - } - run + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.krate("test") } fn make_run(run: RunConfig<'_>) {