From d34a1b0c1b4b22cc61b5956c07d89517bf278af8 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 2 Jul 2020 08:07:56 -0400 Subject: [PATCH 01/12] Don't duplicate builder code - Add Builder::new_internal --- src/bootstrap/builder.rs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index c1e56347ab1..c1c3c2f3ea7 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -501,16 +501,7 @@ impl<'a> Builder<'a> { _ => return None, }; - let builder = Builder { - build, - top_stage: build.config.stage.unwrap_or(2), - kind, - cache: Cache::new(), - stack: RefCell::new(Vec::new()), - time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), - paths: vec![], - }; - + let builder = Self::new_internal(build, kind, vec![]); let builder = &builder; let mut should_run = ShouldRun::new(builder); for desc in Builder::get_step_descriptions(builder.kind) { @@ -535,6 +526,18 @@ impl<'a> Builder<'a> { Some(help) } + fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { + Builder { + build, + top_stage: build.config.stage.unwrap_or(2), + kind, + cache: Cache::new(), + stack: RefCell::new(Vec::new()), + time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), + paths, + } + } + pub fn new(build: &Build) -> Builder<'_> { let (kind, paths) = match build.config.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), @@ -550,15 +553,7 @@ impl<'a> Builder<'a> { Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), }; - Builder { - build, - top_stage: build.config.stage.unwrap_or(2), - kind, - cache: Cache::new(), - stack: RefCell::new(Vec::new()), - time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), - paths: paths.to_owned(), - } + Self::new_internal(build, kind, paths.to_owned()) } pub fn execute_cli(&self) { From 0192fa4786db80d2c9888af98e7ceec47d327887 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 2 Jul 2020 08:08:09 -0400 Subject: [PATCH 02/12] Make the default stage dependent on the subcommand ### x.py build/test: stage 1 I've seen very few people who actually use full stage 2 builds on purpose. These compile rustc and libstd twice and don't give you much more information than a stage 1 build (except in rare cases like https://github.com/rust-lang/rust/pull/68692#discussion_r376392145). For new contributors, this makes the build process even more daunting than it already is. As long as CI is changed to use `--stage 2` I see no downside here. ### x.py bench/dist/install: stage 2 These commands have to do with a finished, optimized version of rustc. It seems very rare to want to use these with a stage 1 build. ### x.py doc: stage 0 Normally when you document things you're just fixing a typo. In this case there is no need to build the whole rust compiler, since the documentation will usually be the same when generated with the beta compiler or with stage 1. Note that for this release cycle only there will be a significant different between stage0 and stage1 docs: https://github.com/rust-lang/rust/pull/73101. However most of the time this will not be the case. --- src/bootstrap/builder.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index c1c3c2f3ea7..5c25cfea9f2 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -527,9 +527,22 @@ impl<'a> Builder<'a> { } fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { + let top_stage = if let Some(explicit_stage) = build.config.stage { + explicit_stage + } else { + // See https://github.com/rust-lang/compiler-team/issues/326 + match kind { + Kind::Doc => 0, + Kind::Build | Kind::Test => 1, + Kind::Bench | Kind::Dist | Kind::Install => 2, + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Kind::Check | Kind::Clippy | Kind::Fix | Kind::Run | Kind::Format => 0, + } + }; Builder { build, - top_stage: build.config.stage.unwrap_or(2), + top_stage, kind, cache: Cache::new(), stack: RefCell::new(Vec::new()), From f7dcfcd45bd019acf8c914c204ccae519c420adc Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 2 Jul 2020 08:40:34 -0400 Subject: [PATCH 03/12] Don't build rustc without std - Set rustc to build only when explicitly asked for This allows building the stage2 rustc artifacts, which nothing depends on. Previously the behavior was as follows (where stageN <-> stage(N-1) artifacts, except for stage0 libstd): - `x.py build --stage 0`: - stage0 libstd - stage1 rustc (but without putting rustc in stage0/) This leaves you without any rustc at all except for the beta compiler (https://github.com/rust-lang/rust/issues/73519). This is never what you want. - `x.py build --stage 1`: - stage0 libstd - stage1 rustc - stage1 libstd - stage1 rustdoc - stage2 rustc This leaves you with a broken stage2 rustc which doesn't even have libcore and is effectively useless. Additionally, it compiles rustc twice, which is not normally what you want. - `x.py build --stage 2`: - stage0 libstd - stage1 rustc - stage1 libstd - stage2 rustc - stage2 rustdoc and tools This builds all tools in release mode. This is the correct usage for CI, but takes far to long for development. Now the behavior is as follows: - `x.py build --stage 0`: - stage0 libstd This is suitable for contributors only working on the standard library, as it means rustc never has to be compiled. - `x.py build --stage 1`: - stage0 libstd - stage1 rustc - stage1 libstd - stage1 rustdoc This is suitable for contributors working on the compiler. It ensures that you have a working rustc and libstd without having to pass `src/libstd` in addition. - `x.py build --stage 2`: - stage0 libstd - stage1 rustc - stage1 libstd - stage2 rustc - stage2 libstd - stage2 rustdoc This is suitable for debugging errors which only appear with the stage2 compiler. - `x.py build --stage 2 src/libstd src/rustc` - stage0 libstd - stage1 rustc - stage1 libstd - stage2 rustc - stage2 libstd - stage2 rustdoc, tools, etc. - stage2 rustc artifacts ('stage3') This is suitable for CI, which wants all tools in release mode. However, most of the use cases for this should use `x.py dist` instead, which builds all the tools without each having to be named individually. --- src/bootstrap/compile.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index e3d1e005373..373e240cb8e 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -446,10 +446,10 @@ pub struct Rustc { impl Step for Rustc { type Output = (); const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; + const DEFAULT: bool = false; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.all_krates("rustc-main") + run.path("src/rustc") } fn make_run(run: RunConfig<'_>) { From 01c6256178fb126d668045f3a1297e0f3491e985 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 2 Jul 2020 08:59:50 -0400 Subject: [PATCH 04/12] Change debuginfo to default to 1 if `debug = true` is set From [a conversation in discord](https://discordapp.com/channels/442252698964721669/443151243398086667/719200989269327882): > Linking seems to consume all available RAM, leading to the OS to swap memory to disk and slowing down everything in the process Compiling itself doesn't seem to take up as much RAM, and I'm only looking to check whether a minimal testcase can be compiled by rustc, where the runtime performance isn't much of an issue > do you have debug = true or debuginfo-level = 2 in config.toml? > if so I think that results in over 2GB of debuginfo nowadays and is likely the culprit > which might mean we're giving out bad advice :( Anecdotally, this sped up my stage 1 build from 15 to 10 minutes. This still adds line numbers, it only removes variable and type information. - Improve wording for debuginfo description Co-authored-by: Teymour Aldridge <42674621+teymour-aldridge@users.noreply.github.com> --- config.toml.example | 5 ++++- src/bootstrap/config.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.toml.example b/config.toml.example index 01952b21ba4..5d198abe1c7 100644 --- a/config.toml.example +++ b/config.toml.example @@ -341,7 +341,10 @@ # Debuginfo for tests run with compiletest is not controlled by this option # and needs to be enabled separately with `debuginfo-level-tests`. # -# Defaults to 2 if debug is true +# Note that debuginfo-level = 2 generates several gigabytes of debuginfo +# and will slow down the linking process significantly. +# +# Defaults to 1 if debug is true #debuginfo-level = 0 # Debuginfo level for the compiler. diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index d71f3170420..d64ca95d243 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -734,7 +734,7 @@ impl Config { let with_defaults = |debuginfo_level_specific: Option| { debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { - 2 + 1 } else { 0 }) From 74b373426a79af2b21b8d266a881753e338cfd2e Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 13 Jul 2020 19:55:33 -0400 Subject: [PATCH 05/12] Fix most bootstrap tests Uses --stage 2 for all the existing tests --- src/bootstrap/builder.rs | 2 +- src/bootstrap/builder/tests.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5c25cfea9f2..f2f0906ff4e 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -232,7 +232,7 @@ impl StepDescription { } if !attempted_run { - panic!("Error: no rules matched {}.", path.display()); + panic!("error: no rules matched {}", path.display()); } } } diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index 6684ca82c27..240553d8baf 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -6,6 +6,7 @@ use pretty_assertions::assert_eq; fn configure(host: &[&str], target: &[&str]) -> Config { let mut config = Config::default_opts(); + config.stage = Some(2); // don't save toolstates config.save_toolstates = None; config.skip_only_host_steps = false; @@ -421,7 +422,7 @@ fn test_exclude() { } #[test] -fn doc_default() { +fn doc_ci() { let mut config = configure(&[], &[]); config.compiler_docs = true; config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; From 60c17297383102964ae08bed7e0d2f202959f7f9 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 13 Jul 2020 20:20:42 -0400 Subject: [PATCH 06/12] Move tests into a submodule --- src/bootstrap/builder/tests.rs | 927 +++++++++++++++++---------------- 1 file changed, 468 insertions(+), 459 deletions(-) diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index 240553d8baf..8e98441afa6 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -2,8 +2,6 @@ use super::*; use crate::config::{Config, TargetSelection}; use std::thread; -use pretty_assertions::assert_eq; - fn configure(host: &[&str], target: &[&str]) -> Config { let mut config = Config::default_opts(); config.stage = Some(2); @@ -36,463 +34,474 @@ fn first(v: Vec<(A, B)>) -> Vec { v.into_iter().map(|(a, _)| a).collect::>() } -#[test] -fn dist_baseline() { - let build = Build::new(configure(&[], &[])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); +mod dist { + use super::{configure, first}; + use crate::builder::*; + use pretty_assertions::assert_eq; - let a = TargetSelection::from_user("A"); + #[test] + fn dist_baseline() { + let build = Build::new(configure(&[], &[])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - assert_eq!(first(builder.cache.all::()), &[dist::Docs { host: a },]); - assert_eq!(first(builder.cache.all::()), &[dist::Mingw { host: a },]); - assert_eq!( - first(builder.cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] - ); - assert_eq!(first(builder.cache.all::()), &[dist::Src]); - // Make sure rustdoc is only built once. - assert_eq!( - first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); -} - -#[test] -fn dist_with_targets() { - let build = Build::new(configure(&[], &["B"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - ] - ); - assert_eq!(first(builder.cache.all::()), &[dist::Src]); -} - -#[test] -fn dist_with_hosts() { - let build = Build::new(configure(&["B"], &[])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!(first(builder.cache.all::()), &[dist::Src]); -} - -#[test] -fn dist_only_cross_host() { - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let mut build = Build::new(configure(&["B"], &[])); - build.config.docs = false; - build.config.extended = true; - build.hosts = vec![b]; - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); -} - -#[test] -fn dist_with_targets_and_hosts() { - let build = Build::new(configure(&["B"], &["C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ] - ); - assert_eq!(first(builder.cache.all::()), &[dist::Src]); -} - -#[test] -fn dist_with_target_flag() { - let mut config = configure(&["B"], &["C"]); - config.skip_only_host_steps = true; // as-if --target=C was passed - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!(first(builder.cache.all::()), &[]); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ] - ); - assert_eq!(first(builder.cache.all::()), &[]); -} - -#[test] -fn dist_with_same_targets_and_hosts() { - let build = Build::new(configure(&["B"], &["B"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(builder.cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!(first(builder.cache.all::()), &[dist::Src]); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, - ] - ); -} - -#[test] -fn build_default() { - let build = Build::new(configure(&["B"], &["C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, - ] - ); - assert!(!builder.cache.all::().is_empty()); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, - compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: b }, - ] - ); -} - -#[test] -fn build_with_target_flag() { - let mut config = configure(&["B"], &["C"]); - config.skip_only_host_steps = true; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); -} - -#[test] -fn test_with_no_doc_stage0() { - let mut config = configure(&[], &[]); - config.stage = Some(0); - config.cmd = Subcommand::Test { - paths: vec!["library/std".into()], - test_args: vec![], - rustc_args: vec![], - fail_fast: true, - doc_tests: DocTests::No, - bless: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - }; - - let build = Build::new(config); - let mut builder = Builder::new(&build); - - let host = TargetSelection::from_user("A"); - - builder - .run_step_descriptions(&[StepDescription::from::()], &["library/std".into()]); - - // Ensure we don't build any compiler artifacts. - assert!(!builder.cache.contains::()); - assert_eq!( - first(builder.cache.all::()), - &[test::Crate { - compiler: Compiler { host, stage: 0 }, - target: host, - mode: Mode::Std, - test_kind: test::TestKind::Test, - krate: INTERNER.intern_str("std"), - },] - ); -} - -#[test] -fn test_exclude() { - let mut config = configure(&[], &[]); - config.exclude = vec!["src/tools/tidy".into()]; - config.cmd = Subcommand::Test { - paths: Vec::new(), - test_args: Vec::new(), - rustc_args: Vec::new(), - fail_fast: true, - doc_tests: DocTests::No, - bless: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - }; - - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - - // Ensure we have really excluded tidy - assert!(!builder.cache.contains::()); - - // Ensure other tests are not affected. - assert!(builder.cache.contains::()); -} - -#[test] -fn doc_ci() { - let mut config = configure(&[], &[]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // This is actually stage 1, but Rustdoc::run swaps out the compiler with - // stage minus 1 if --stage is not 0. Very confusing! - assert_eq!( - first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); -} - -//FIXME(mark-i-m): reinstate this test when things are fixed... -//#[test] -#[allow(dead_code)] -fn test_docs() { - // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&[], &[]); - config.cmd = Subcommand::Test { - paths: vec![], - test_args: vec![], - rustc_args: vec![], - fail_fast: true, - doc_tests: DocTests::Yes, - bless: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // Unfortunately rustdoc is built twice. Once from stage1 for compiletest - // (and other things), and once from stage0 for std crates. Ideally it - // would only be built once. If someone wants to fix this, it might be - // worth investigating if it would be possible to test std from stage1. - // Note that the stages here are +1 than what they actually are because - // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is - // not 0. - assert_eq!( - first(builder.cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ] - ); + let a = TargetSelection::from_user("A"); + + assert_eq!(first(builder.cache.all::()), &[dist::Docs { host: a },]); + assert_eq!(first(builder.cache.all::()), &[dist::Mingw { host: a },]); + assert_eq!( + first(builder.cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!(first(builder.cache.all::()), &[dist::Src]); + // Make sure rustdoc is only built once. + assert_eq!( + first(builder.cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn dist_with_targets() { + let build = Build::new(configure(&[], &["B"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ] + ); + assert_eq!(first(builder.cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_hosts() { + let build = Build::new(configure(&["B"], &[])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!(first(builder.cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_only_cross_host() { + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let mut build = Build::new(configure(&["B"], &[])); + build.config.docs = false; + build.config.extended = true; + build.hosts = vec![b]; + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + } + + #[test] + fn dist_with_targets_and_hosts() { + let build = Build::new(configure(&["B"], &["C"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + ] + ); + assert_eq!(first(builder.cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_target_flag() { + let mut config = configure(&["B"], &["C"]); + config.skip_only_host_steps = true; // as-if --target=C was passed + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] + ); + assert_eq!(first(builder.cache.all::()), &[]); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + ] + ); + assert_eq!(first(builder.cache.all::()), &[]); + } + + #[test] + fn dist_with_same_targets_and_hosts() { + let build = Build::new(configure(&["B"], &["B"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(builder.cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!(first(builder.cache.all::()), &[dist::Src]); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, + ] + ); + } + + #[test] + fn build_default() { + let build = Build::new(configure(&["B"], &["C"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions( + &Builder::get_step_descriptions(Kind::Build), + &["src/rustc".into(), "src/libstd".into()], + ); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, + ] + ); + assert!(!builder.cache.all::().is_empty()); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: a }, + compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, + compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: b }, + compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: b }, + ] + ); + } + + #[test] + fn build_with_target_flag() { + let mut config = configure(&["B"], &["C"]); + config.skip_only_host_steps = true; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + } + + #[test] + fn test_with_no_doc_stage0() { + let mut config = configure(&[], &[]); + config.stage = Some(0); + config.cmd = Subcommand::Test { + paths: vec!["library/std".into()], + test_args: vec![], + rustc_args: vec![], + fail_fast: true, + doc_tests: DocTests::No, + bless: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + }; + + let build = Build::new(config); + let mut builder = Builder::new(&build); + + let host = TargetSelection::from_user("A"); + + builder.run_step_descriptions( + &[StepDescription::from::()], + &["library/std".into()], + ); + + // Ensure we don't build any compiler artifacts. + assert!(!builder.cache.contains::()); + assert_eq!( + first(builder.cache.all::()), + &[test::Crate { + compiler: Compiler { host, stage: 0 }, + target: host, + mode: Mode::Std, + test_kind: test::TestKind::Test, + krate: INTERNER.intern_str("std"), + },] + ); + } + + #[test] + fn test_exclude() { + let mut config = configure(&[], &[]); + config.exclude = vec!["src/tools/tidy".into()]; + config.cmd = Subcommand::Test { + paths: Vec::new(), + test_args: Vec::new(), + rustc_args: Vec::new(), + fail_fast: true, + doc_tests: DocTests::No, + bless: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + }; + + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); + + // Ensure we have really excluded tidy + assert!(!builder.cache.contains::()); + + // Ensure other tests are not affected. + assert!(builder.cache.contains::()); + } + + #[test] + fn doc_ci() { + let mut config = configure(&[], &[]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // This is actually stage 1, but Rustdoc::run swaps out the compiler with + // stage minus 1 if --stage is not 0. Very confusing! + assert_eq!( + first(builder.cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + //FIXME(mark-i-m): reinstate this test when things are fixed... + //#[test] + #[allow(dead_code)] + fn test_docs() { + // Behavior of `x.py test` doing various documentation tests. + let mut config = configure(&[], &[]); + config.cmd = Subcommand::Test { + paths: vec![], + test_args: vec![], + rustc_args: vec![], + fail_fast: true, + doc_tests: DocTests::Yes, + bless: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // Unfortunately rustdoc is built twice. Once from stage1 for compiletest + // (and other things), and once from stage0 for std crates. Ideally it + // would only be built once. If someone wants to fix this, it might be + // worth investigating if it would be possible to test std from stage1. + // Note that the stages here are +1 than what they actually are because + // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is + // not 0. + assert_eq!( + first(builder.cache.all::()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] + ); + } } From cdca33754771e6b10e9399c8d7780e391963305e Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 13 Jul 2020 20:25:21 -0400 Subject: [PATCH 07/12] Add tests for the new behavior - Only set stage 2 in dist tests - Add test for `x.py doc` without args - Add test for `x.py build` without args - Add test for `x.py build --stage 0` --- src/bootstrap/builder/tests.rs | 95 ++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index 8e98441afa6..d6056565356 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -4,7 +4,6 @@ use std::thread; fn configure(host: &[&str], target: &[&str]) -> Config { let mut config = Config::default_opts(); - config.stage = Some(2); // don't save toolstates config.save_toolstates = None; config.skip_only_host_steps = false; @@ -34,11 +33,101 @@ fn first(v: Vec<(A, B)>) -> Vec { v.into_iter().map(|(a, _)| a).collect::>() } -mod dist { +mod defaults { use super::{configure, first}; use crate::builder::*; + use crate::Config; use pretty_assertions::assert_eq; + #[test] + fn build_default() { + let build = Build::new(configure(&[], &[])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + ] + ); + assert!(!builder.cache.all::().is_empty()); + // Make sure rustdoc is only built once. + assert_eq!( + first(builder.cache.all::()), + // Recall that rustdoc stages are off-by-one + // - this is the compiler it's _linked_ to, not built with. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], + ); + assert_eq!( + first(builder.cache.all::()), + &[compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },] + ); + } + + #[test] + fn build_stage_0() { + let config = Config { stage: Some(0), ..configure(&[], &[]) }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + assert_eq!( + first(builder.cache.all::()), + &[compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },] + ); + assert!(!builder.cache.all::().is_empty()); + assert_eq!( + first(builder.cache.all::()), + // This is the beta rustdoc. + // Add an assert here to make sure this is the only rustdoc built. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], + ); + assert!(builder.cache.all::().is_empty()); + } + + #[test] + fn doc_default() { + let mut config = configure(&[], &[]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 0 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 0 }, target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] + ); + // docs should be built with the beta compiler, not with the stage0 artifacts. + // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, + // not the one it was built by. + assert_eq!( + first(builder.cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] + ); + } +} + +mod dist { + use super::{first, Config}; + use crate::builder::*; + use pretty_assertions::assert_eq; + + fn configure(host: &[&str], target: &[&str]) -> Config { + Config { stage: Some(2), ..super::configure(host, target) } + } + #[test] fn dist_baseline() { let build = Build::new(configure(&[], &[])); @@ -276,7 +365,7 @@ mod dist { } #[test] - fn build_default() { + fn build_all() { let build = Build::new(configure(&["B"], &["C"])); let mut builder = Builder::new(&build); builder.run_step_descriptions( From 4ee8d847e5f411e178fa5224d16dae38affb9392 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 13 Jul 2020 22:28:26 -0400 Subject: [PATCH 08/12] Use --stage 2 explicitly in CI - expand yaml anchors - don't use --stage 2 for dist; that's already the default --- .github/workflows/ci.yml | 6 +++--- src/ci/azure-pipelines/auto.yml | 2 +- src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile | 2 +- src/ci/docker/host-x86_64/arm-android/Dockerfile | 2 +- src/ci/docker/host-x86_64/armhf-gnu/Dockerfile | 2 +- src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile | 2 +- .../host-x86_64/disabled/dist-armv7-android/Dockerfile | 2 +- .../host-x86_64/disabled/dist-i686-android/Dockerfile | 2 +- .../host-x86_64/disabled/riscv64gc-linux/Dockerfile | 2 +- .../host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile | 2 +- src/ci/docker/host-x86_64/dist-various-1/Dockerfile | 2 +- src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile | 2 +- src/ci/docker/host-x86_64/i686-gnu/Dockerfile | 2 +- src/ci/docker/host-x86_64/mingw-check/Dockerfile | 4 ++-- src/ci/docker/host-x86_64/test-various/Dockerfile | 6 +++--- src/ci/docker/host-x86_64/wasm32/Dockerfile | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile | 4 ++-- src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile | 4 ++-- .../host-x86_64/x86_64-gnu-full-bootstrap/Dockerfile | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile | 8 ++++---- src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 2 +- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 6 +++--- 24 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d83971b70be..9b35208bf32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -406,7 +406,7 @@ jobs: os: windows-latest-xl - name: x86_64-msvc-cargo env: - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-lld" VCVARS_BAT: vcvars64.bat NO_DEBUG_ASSERTIONS: 1 @@ -414,7 +414,7 @@ jobs: os: windows-latest-xl - name: x86_64-msvc-tools env: - SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py --stage 2 /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json" os: windows-latest-xl - name: i686-mingw-1 @@ -598,7 +598,7 @@ jobs: os: macos-latest - name: x86_64-apple env: - SCRIPT: "./x.py test" + SCRIPT: "./x.py --stage 2 test" RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.8 diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 06e284c763c..2185b0d30db 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -36,7 +36,7 @@ jobs: # Note that the compiler is compiled to target 10.8 here because the Xcode # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. x86_64-apple: - SCRIPT: ./x.py test + SCRIPT: ./x.py --stage 2 test INITIAL_RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.8 diff --git a/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile b/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile index 114ac832cf5..b6cf60a5e15 100644 --- a/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile +++ b/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile @@ -23,4 +23,4 @@ ENV RUST_CONFIGURE_ARGS \ --enable-sanitizers \ --enable-profiler \ --enable-compiler-docs -ENV SCRIPT python3 ../x.py test +ENV SCRIPT python3 ../x.py --stage 2 test diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index aa9335c473b..add2647fa1e 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -31,7 +31,7 @@ ENV TARGETS=arm-linux-androideabi ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 -ENV SCRIPT python3 ../x.py test --target $TARGETS +ENV SCRIPT python3 ../x.py --stage 2 test --target $TARGETS COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile index 71071761f05..1f3092c5513 100644 --- a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile @@ -78,6 +78,6 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs -ENV SCRIPT python3 ../x.py test --target arm-unknown-linux-gnueabihf +ENV SCRIPT python3 ../x.py --stage 2 test --target arm-unknown-linux-gnueabihf ENV NO_CHANGE_USER=1 diff --git a/src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile b/src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile index 3fa65511e94..a4d9f53ebab 100644 --- a/src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile +++ b/src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile @@ -33,7 +33,7 @@ ENV EMCC_CFLAGS=-O1 # Emscripten installation is user-specific ENV NO_CHANGE_USER=1 -ENV SCRIPT python3 ../x.py test --target $TARGETS +ENV SCRIPT python3 ../x.py --stage 2 test --target $TARGETS # This is almost identical to the wasm32-unknown-emscripten target, so # running with assertions again is not useful diff --git a/src/ci/docker/host-x86_64/disabled/dist-armv7-android/Dockerfile b/src/ci/docker/host-x86_64/disabled/dist-armv7-android/Dockerfile index 7227c41ccca..f986c38ea02 100644 --- a/src/ci/docker/host-x86_64/disabled/dist-armv7-android/Dockerfile +++ b/src/ci/docker/host-x86_64/disabled/dist-armv7-android/Dockerfile @@ -33,7 +33,7 @@ ENV RUST_CONFIGURE_ARGS \ # build to finish we use --warn-unresolved-symbols. Note that the missing # symbols does not affect std, only the compiler (llvm) and cargo (openssl). ENV SCRIPT \ - python3 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \ + python3 ../x.py --stage 2 build src/llvm --host $HOSTS --target $HOSTS && \ (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \ rm /android/ndk/arm && \ ln -s /android/ndk/arm-14 /android/ndk/arm && \ diff --git a/src/ci/docker/host-x86_64/disabled/dist-i686-android/Dockerfile b/src/ci/docker/host-x86_64/disabled/dist-i686-android/Dockerfile index b74dcefa351..4dfbc725607 100644 --- a/src/ci/docker/host-x86_64/disabled/dist-i686-android/Dockerfile +++ b/src/ci/docker/host-x86_64/disabled/dist-i686-android/Dockerfile @@ -33,7 +33,7 @@ ENV RUST_CONFIGURE_ARGS \ # build to finish we use --warn-unresolved-symbols. Note that the missing # symbols does not affect std, only the compiler (llvm) and cargo (openssl). ENV SCRIPT \ - python3 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \ + python3 ../x.py --stage 2 build src/llvm --host $HOSTS --target $HOSTS && \ (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \ rm /android/ndk/x86 && \ ln -s /android/ndk/x86-14 /android/ndk/x86 && \ diff --git a/src/ci/docker/host-x86_64/disabled/riscv64gc-linux/Dockerfile b/src/ci/docker/host-x86_64/disabled/riscv64gc-linux/Dockerfile index a938899636a..e3c35000eb8 100644 --- a/src/ci/docker/host-x86_64/disabled/riscv64gc-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/disabled/riscv64gc-linux/Dockerfile @@ -97,6 +97,6 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --qemu-riscv64-rootfs=/tmp/rootfs -ENV SCRIPT python3 ../x.py test --target riscv64gc-unknown-linux-gnu +ENV SCRIPT python3 ../x.py --stage 2 test --target riscv64gc-unknown-linux-gnu ENV NO_CHANGE_USER=1 diff --git a/src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile b/src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile index 996fffeb871..162d7a1345c 100644 --- a/src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile @@ -46,5 +46,5 @@ ENV CFLAGS_i586_unknown_linux_musl=-Wa,-mrelax-relocations=no ENV TARGETS=i586-unknown-linux-gnu,i686-unknown-linux-musl ENV SCRIPT \ - python3 ../x.py test --target $TARGETS && \ + python3 ../x.py --stage 2 test --target $TARGETS && \ python3 ../x.py dist --target $TARGETS,i586-unknown-linux-musl diff --git a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile index ac228cfe01d..fdd777e824b 100644 --- a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile @@ -192,7 +192,7 @@ ENV RUST_CONFIGURE_ARGS \ --disable-docs ENV SCRIPT \ - python3 ../x.py test --target $RUN_MAKE_TARGETS src/test/run-make && \ + python3 ../x.py --stage 2 test --target $RUN_MAKE_TARGETS src/test/run-make && \ python3 ../x.py dist --target $TARGETS # sccache diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile index 436215839f7..6a596b3465f 100644 --- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile @@ -20,7 +20,7 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests -ENV SCRIPT python3 ../x.py test +ENV SCRIPT python3 ../x.py --stage 2 test # FIXME(#59637) takes too long on CI right now ENV NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile index 34a76f39668..9d319017d79 100644 --- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile @@ -22,7 +22,7 @@ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu # Exclude some tests that are unlikely to be platform specific, to speed up # this slow job. -ENV SCRIPT python3 ../x.py test \ +ENV SCRIPT python3 ../x.py --stage 2 test \ --exclude src/bootstrap \ --exclude src/test/rustdoc-js \ --exclude src/tools/error_index_generator \ diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 0c59b95ea21..b902eda87bc 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -22,10 +22,10 @@ RUN sh /scripts/sccache.sh COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 -ENV SCRIPT python3 ../x.py test src/tools/expand-yaml-anchors && \ +ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python3 ../x.py build --stage 0 src/tools/build-manifest && \ python3 ../x.py test --stage 0 src/tools/compiletest && \ - python3 ../x.py test src/tools/tidy && \ + python3 ../x.py test --stage 2 src/tools/tidy && \ python3 ../x.py doc --stage 0 library/std && \ /scripts/validate-toolstate.sh diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile index b7276f60867..c55a284e137 100644 --- a/src/ci/docker/host-x86_64/test-various/Dockerfile +++ b/src/ci/docker/host-x86_64/test-various/Dockerfile @@ -40,7 +40,7 @@ ENV RUST_CONFIGURE_ARGS \ ENV NO_DEBUG_ASSERTIONS=1 ENV WASM_TARGETS=wasm32-unknown-unknown -ENV WASM_SCRIPT python3 /checkout/x.py test --target $WASM_TARGETS \ +ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --target $WASM_TARGETS \ src/test/run-make \ src/test/ui \ src/test/compile-fail \ @@ -49,13 +49,13 @@ ENV WASM_SCRIPT python3 /checkout/x.py test --target $WASM_TARGETS \ library/core ENV NVPTX_TARGETS=nvptx64-nvidia-cuda -ENV NVPTX_SCRIPT python3 /checkout/x.py test --target $NVPTX_TARGETS \ +ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --target $NVPTX_TARGETS \ src/test/run-make \ src/test/assembly ENV MUSL_TARGETS=x86_64-unknown-linux-musl \ CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \ CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++ -ENV MUSL_SCRIPT python3 /checkout/x.py test --target $MUSL_TARGETS +ENV MUSL_SCRIPT python3 /checkout/x.py --stage 2 test --target $MUSL_TARGETS ENV SCRIPT $WASM_SCRIPT && $NVPTX_SCRIPT && $MUSL_SCRIPT diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile index a40ccb6bfd5..e00177b4a67 100644 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ b/src/ci/docker/host-x86_64/wasm32/Dockerfile @@ -52,7 +52,7 @@ ENV NO_CHANGE_USER=1 # FIXME: Re-enable these tests once https://github.com/rust-lang/cargo/pull/7476 # is picked up by CI -ENV SCRIPT python3 ../x.py test --target $TARGETS \ +ENV SCRIPT python3 ../x.py test --stage 2 --target $TARGETS \ --exclude library/core \ --exclude library/alloc \ --exclude library/proc_macro \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile index c5e41b8a75a..c1cb20b631d 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile @@ -40,5 +40,5 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-linux-gnu.cxx=clang++ ENV SCRIPT \ - python3 ../x.py build && \ - python3 ../x.py test src/test/run-make-fulldeps --test-args clang + python3 ../x.py --stage 2 build && \ + python3 ../x.py --stage 2 test src/test/run-make-fulldeps --test-args clang diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile index cc07a591cc1..68e89a7bade 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile @@ -19,10 +19,10 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false -ENV SCRIPT python3 ../x.py test distcheck +ENV SCRIPT python3 ../x.py --stage 2 test distcheck ENV DIST_SRC 1 -# The purpose of this builder is to test that we can `./x.py test` successfully +# The purpose of this builder is to test that we can `./x.py --stage 2 test` successfully # from a tarball, not to test LLVM/rustc's own set of assertions. These cause a # significant hit to CI compile time (over a half hour as observed in #61185), # so disable assertions for this builder. diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-full-bootstrap/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-full-bootstrap/Dockerfile index de7ee6950b5..8648e5ed7a4 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-full-bootstrap/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-full-bootstrap/Dockerfile @@ -21,7 +21,7 @@ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-full-bootstrap -ENV SCRIPT python3 ../x.py build +ENV SCRIPT python3 ../x.py --stage 2 build # In general this just slows down the build and we're just a smoke test that # a full bootstrap works in general, so there's not much need to take this diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile index 1d9cad149d9..5c971c73c97 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile @@ -30,7 +30,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-llvm-link-shared \ --set rust.thin-lto-import-instr-limit=10 -ENV SCRIPT python2.7 ../x.py test --exclude src/tools/tidy && \ +ENV SCRIPT python2.7 ../x.py --stage 2 test --exclude src/tools/tidy && \ # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have # both 32-bit and 64-bit outputs updated by the PR author, before @@ -43,7 +43,7 @@ ENV SCRIPT python2.7 ../x.py test --exclude src/tools/tidy && \ # This also requires `--pass=build` because we can't execute the tests # on the `x86_64` host when they're built as `armv5te` binaries. # (we're only interested in the MIR output, so this doesn't matter) - python2.7 ../x.py test src/test/mir-opt --pass=build \ + python2.7 ../x.py --stage 2 test src/test/mir-opt --pass=build \ --target=armv5te-unknown-linux-gnueabi && \ # Run the UI test suite again, but in `--pass=check` mode # @@ -53,9 +53,9 @@ ENV SCRIPT python2.7 ../x.py test --exclude src/tools/tidy && \ # FIXME: We ideally want to test this in 32-bit mode, but currently # (due to the LLVM problems mentioned above) that isn't readily # possible. - python2.7 ../x.py test src/test/ui --pass=check && \ + python2.7 ../x.py --stage 2 test src/test/ui --pass=check && \ # Run tidy at the very end, after all the other tests. - python2.7 ../x.py test src/tools/tidy + python2.7 ../x.py --stage 2 test src/tools/tidy # The purpose of this container isn't to test with debug assertions and # this is run on all PRs, so let's get speedier builds by disabling these extra diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile index 096f67e13d1..fa769cac9c1 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile @@ -21,4 +21,4 @@ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \ --disable-optimize-tests \ --set rust.test-compare-mode -ENV SCRIPT python3 ../x.py test +ENV SCRIPT python3 ../x.py --stage 2 test diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index ef17f0507ab..e99b3f6ec37 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -22,4 +22,4 @@ COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --save-toolstates=/tmp/toolstate/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py +ENV SCRIPT /tmp/checktools.sh ../x.py --stage 2 diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index af6e1318062..f8bacf79ac0 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -23,4 +23,4 @@ ENV RUST_CONFIGURE_ARGS \ --enable-sanitizers \ --enable-profiler \ --enable-compiler-docs -ENV SCRIPT python3 ../x.py test +ENV SCRIPT python3 ../x.py --stage 2 test diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 0aeb6a04e5f..b31f00c0322 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -471,7 +471,7 @@ jobs: - name: x86_64-msvc-cargo env: - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld VCVARS_BAT: vcvars64.bat # FIXME(#59637) @@ -481,7 +481,7 @@ jobs: - name: x86_64-msvc-tools env: - SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py --stage 2 /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json <<: *job-windows-xl @@ -613,7 +613,7 @@ jobs: - name: x86_64-apple env: - SCRIPT: ./x.py test + SCRIPT: ./x.py --stage 2 test RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.8 From 7768eaa0503e7e98eb2404fa4664297a84d72d38 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 27 Jul 2020 15:53:01 -0400 Subject: [PATCH 09/12] Add assert that tests happen with stage 2 in CI - Use stage 2 for makefile - Move assert to builder - Don't add an assert for --help - Allow --stage 0 if passed explicitly - Don't assert defaults during tests Otherwise it's impossible to test the defaults! --- src/bootstrap/builder.rs | 16 +++++++++++++++- src/bootstrap/mk/Makefile.in | 34 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index f2f0906ff4e..a7add4cd559 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -540,6 +540,7 @@ impl<'a> Builder<'a> { Kind::Check | Kind::Clippy | Kind::Fix | Kind::Run | Kind::Format => 0, } }; + Builder { build, top_stage, @@ -566,7 +567,20 @@ impl<'a> Builder<'a> { Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), }; - Self::new_internal(build, kind, paths.to_owned()) + let this = Self::new_internal(build, kind, paths.to_owned()); + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if build.config.stage.is_none() && build.ci_env != crate::CiEnv::None { + match kind { + Kind::Test | Kind::Doc | Kind::Build | Kind::Bench | Kind::Dist | Kind::Install => { + assert_eq!(this.top_stage, 2) + } + _ => {} + } + } + + this } pub fn execute_cli(&self) { diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index e5b9f27c258..1564cfb0619 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -9,8 +9,8 @@ endif BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py all: - $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) - $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) build --stage 2 $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS) help: $(Q)echo 'Welcome to the rustbuild build system!' @@ -31,17 +31,17 @@ rustc-stage2: docs: doc doc: - $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS) nomicon: - $(Q)$(BOOTSTRAP) doc src/doc/nomicon $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/nomicon $(BOOTSTRAP_ARGS) book: - $(Q)$(BOOTSTRAP) doc src/doc/book $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/book $(BOOTSTRAP_ARGS) standalone-docs: - $(Q)$(BOOTSTRAP) doc src/doc $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc $(BOOTSTRAP_ARGS) check: - $(Q)$(BOOTSTRAP) test $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) test --stage 2 $(BOOTSTRAP_ARGS) check-aux: - $(Q)$(BOOTSTRAP) test \ + $(Q)$(BOOTSTRAP) test --stage 2 \ src/tools/cargo \ src/tools/cargotest \ $(BOOTSTRAP_ARGS) @@ -51,18 +51,18 @@ dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) distcheck: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) - $(Q)$(BOOTSTRAP) test distcheck $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) test --stage 2 distcheck $(BOOTSTRAP_ARGS) install: $(Q)$(BOOTSTRAP) install $(BOOTSTRAP_ARGS) tidy: - $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) test --stage 2 src/tools/tidy $(BOOTSTRAP_ARGS) prepare: - $(Q)$(BOOTSTRAP) build nonexistent/path/to/trigger/cargo/metadata + $(Q)$(BOOTSTRAP) build --stage 2 nonexistent/path/to/trigger/cargo/metadata check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: - $(Q)$(BOOTSTRAP) test --target arm-linux-androideabi + $(Q)$(BOOTSTRAP) test --stage 2 --target arm-linux-androideabi check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: - $(Q)$(BOOTSTRAP) test --target x86_64-unknown-linux-musl + $(Q)$(BOOTSTRAP) test --stage 2 --target x86_64-unknown-linux-musl TESTS_IN_2 := \ src/test/ui \ @@ -70,18 +70,18 @@ TESTS_IN_2 := \ src/tools/linkchecker ci-subset-1: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %) ci-subset-2: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2) TESTS_IN_MINGW_2 := \ src/test/ui \ src/test/compile-fail ci-mingw-subset-1: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_MINGW_2:%=--exclude %) + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) ci-mingw-subset-2: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_MINGW_2) + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2) .PHONY: dist From a5337d668c4976fed89551f94b03b2903f8f947c Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 27 Jul 2020 21:53:10 -0400 Subject: [PATCH 10/12] Use exhaustive match for assert --- src/bootstrap/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index a7add4cd559..144e146685f 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -576,7 +576,7 @@ impl<'a> Builder<'a> { Kind::Test | Kind::Doc | Kind::Build | Kind::Bench | Kind::Dist | Kind::Install => { assert_eq!(this.top_stage, 2) } - _ => {} + Kind::Check | Kind::Clippy | Kind::Fix | Kind::Run | Kind::Format => {} } } From c4c6453b7ba42c5114e875908eeaf510398d2f38 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Tue, 28 Jul 2020 08:34:59 -0400 Subject: [PATCH 11/12] Fix bad rebase --- src/bootstrap/builder/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index d6056565356..4293844fcbf 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -370,7 +370,7 @@ mod dist { let mut builder = Builder::new(&build); builder.run_step_descriptions( &Builder::get_step_descriptions(Kind::Build), - &["src/rustc".into(), "src/libstd".into()], + &["src/rustc".into(), "library/std".into()], ); let a = TargetSelection::from_user("A"); From da40cf81e63d04dced9d943743bb2a58801916e7 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Tue, 28 Jul 2020 09:36:56 -0400 Subject: [PATCH 12/12] Use --stage 2 in checktools - Remove useless --stage 2 argument to checktools.sh - Fix help text for expand-yaml-anchors (it had a typo) --- .github/workflows/ci.yml | 2 +- src/bootstrap/flags.rs | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh | 6 +++--- src/ci/github-actions/ci.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b35208bf32..565c916db5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -414,7 +414,7 @@ jobs: os: windows-latest-xl - name: x86_64-msvc-tools env: - SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py --stage 2 /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json" os: windows-latest-xl - name: i686-mingw-1 diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a298b299667..56e4f0467cc 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -425,7 +425,7 @@ Arguments: This subcommand accepts a number of paths to tools to build and run. For example: - ./x.py run src/tool/expand-yaml-anchors + ./x.py run src/tools/expand-yaml-anchors At least a tool needs to be called.", ); diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index e99b3f6ec37..ef17f0507ab 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -22,4 +22,4 @@ COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --save-toolstates=/tmp/toolstate/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py --stage 2 +ENV SCRIPT /tmp/checktools.sh ../x.py diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index b4b23a245e0..49a8e5e88a0 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -7,7 +7,7 @@ X_PY="$1" # Try to test all the tools and store the build/test success in the TOOLSTATE_FILE set +e -python3 "$X_PY" test --no-fail-fast \ +python3 "$X_PY" test --stage 2 --no-fail-fast \ src/doc/book \ src/doc/nomicon \ src/doc/reference \ @@ -22,5 +22,5 @@ set -e # debugging: print out the saved toolstates cat /tmp/toolstate/toolstates.json -python3 "$X_PY" test check-tools -python3 "$X_PY" test src/tools/clippy +python3 "$X_PY" test --stage 2 check-tools +python3 "$X_PY" test --stage 2 src/tools/clippy diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index b31f00c0322..a7c1987e8b1 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -481,7 +481,7 @@ jobs: - name: x86_64-msvc-tools env: - SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py --stage 2 /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json <<: *job-windows-xl