From 645784521950c51260abf68c7be8d24552430338 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 22 Jan 2021 00:31:17 -0500 Subject: [PATCH] Absolute bare minimum for downloading rustc from CI - Use the same compiler for stage0 and stage1. This should be fixed at some point (so bootstrap isn't constantly rebuilt). - Make sure `x.py build` and `x.py check` work. - Use `git merge-base` to determine the most recent commit to download. - Copy stage0 to the various sysroots in `Sysroot`, and delegate to Sysroot in Assemble. Leave all other code unchanged. - Rename date -> key This can also be a commit hash, so 'date' is no longer a good name. - Add the commented-out option to config.toml.example - Disable all steps by default when `download-rustc` is enabled Most steps don't make sense when downloading a compiler, because they'll be pre-built in the sysroot. Only enable the ones that might be useful, in particular Rustdoc and all `check` steps. At some point, this should probably enable other tools, but rustdoc is enough to test out `download-rustc`. - Don't print 'Skipping' twice in a row Bootstrap forcibly enables a dry run if it isn't already set, so previously it would print the message twice: ``` Skipping bootstrap::compile::Std because it is not enabled for `download-rustc` Skipping bootstrap::compile::Std because it is not enabled for `download-rustc` ``` Now it correctly only prints once. ## Future work - Add FIXME about supporting beta commits - Debug logging will never work. This should be fixed. --- config.toml.example | 6 +++ src/bootstrap/bootstrap.py | 90 +++++++++++++++++++++++++++++++++----- src/bootstrap/builder.rs | 26 ++++++++++- src/bootstrap/check.rs | 4 ++ src/bootstrap/compile.rs | 26 +++++++++-- src/bootstrap/config.rs | 3 ++ src/bootstrap/tool.rs | 1 + 7 files changed, 138 insertions(+), 18 deletions(-) diff --git a/config.toml.example b/config.toml.example index 55b20adabd0..f3bc98d78ae 100644 --- a/config.toml.example +++ b/config.toml.example @@ -358,6 +358,12 @@ changelog-seen = 2 # #debug = false +# Whether to download the stage 1 and 2 compilers from CI. +# This is mostly useful for tools; if you have changes to `compiler/` they will be ignored. +# +# FIXME: currently, this also uses the downloaded compiler for stage0, but that causes unnecessary rebuilds. +#download-rustc = false + # Number of codegen units to use for each compiler invocation. A value of 0 # means "the number of cores on this machine", and 1+ is passed through to the # compiler. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 6708b27b505..bd02b8354ef 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -378,6 +378,7 @@ class RustBuild(object): self.verbose = False self.git_version = None self.nix_deps_dir = None + self.rustc_commit = None def download_stage0(self): """Fetch the build system for Rust, written in Rust @@ -394,20 +395,27 @@ class RustBuild(object): if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or - self.program_out_of_date(self.rustc_stamp(), self.date)): + self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))): if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) + download_rustc = self.rustc_commit is not None tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "rust-std-{}-{}{}".format( rustc_channel, self.build, tarball_suffix) pattern = "rust-std-{}".format(self.build) - self._download_stage0_helper(filename, pattern, tarball_suffix) + self._download_component_helper(filename, pattern, tarball_suffix, download_rustc) filename = "rustc-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - self._download_stage0_helper(filename, "rustc", tarball_suffix) + self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc) filename = "cargo-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - self._download_stage0_helper(filename, "cargo", tarball_suffix) + self._download_component_helper(filename, "cargo", tarball_suffix) + if self.rustc_commit is not None: + filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) + self._download_component_helper( + filename, "rustc-dev", tarball_suffix, download_rustc + ) + self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root())) self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root())) self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root())) @@ -416,7 +424,7 @@ class RustBuild(object): if lib.endswith(".so"): self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True) with output(self.rustc_stamp()) as rust_stamp: - rust_stamp.write(self.date) + rust_stamp.write(self.date + str(self.rustc_commit)) if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and ( not os.path.exists(self.rustfmt()) @@ -426,7 +434,9 @@ class RustBuild(object): tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' [channel, date] = rustfmt_channel.split('-', 1) filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) - self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date) + self._download_component_helper( + filename, "rustfmt-preview", tarball_suffix, key=date + ) self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root())) self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root())) with output(self.rustfmt_stamp()) as rustfmt_stamp: @@ -482,18 +492,27 @@ class RustBuild(object): return opt == "true" \ or (opt == "if-available" and self.build in supported_platforms) - def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None): - if date is None: - date = self.date + def _download_component_helper( + self, filename, pattern, tarball_suffix, download_rustc=False, key=None + ): + if key is None: + if download_rustc: + key = self.rustc_commit + else: + key = self.date cache_dst = os.path.join(self.build_dir, "cache") - rustc_cache = os.path.join(cache_dst, date) + rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): os.makedirs(rustc_cache) - url = "{}/dist/{}".format(self._download_url, date) + if download_rustc: + url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit) + else: + url = "{}/dist/{}".format(self._download_url, key) tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - get("{}/{}".format(url, filename), tarball, verbose=self.verbose) + do_verify = not download_rustc + get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) def _download_ci_llvm(self, llvm_sha, llvm_assertions): @@ -613,6 +632,46 @@ class RustBuild(object): print("warning: failed to call patchelf:", reason) return + # Return the stage1 compiler to download, if any. + def maybe_download_rustc(self): + # If `download-rustc` is not set, default to rebuilding. + if self.get_toml("download-rustc", section="rust") != "true": + return None + # Look for a version to compare to based on the current commit. + # There are a few different cases to handle. + # 1. This commit is a fast-forward from master: `master - * - * - HEAD` + # 2. This commit and master have diverged: + # ``` + # Y - * - HEAD + # / + # X - * - master + # ``` + # In this case, we should compare to `X`. + # 3. `master` and `HEAD` are radically different (>100 commits, or similar). This probably + # means that `master` does *not* correspond to the version we want to compare to, e.g. a + # fork. Instead, we want to compare to `rust-lang/rust:master`, which this has to share a + # recent merge base with. + + # Find which remote corresponds to `rust-lang/rust`. + remotes = subprocess.check_output(["git", "remote", "-v"], universal_newlines=True) + # e.g. `origin https://github.com//rust-lang/rust (fetch)` + rust_lang_remote = next(line for line in remotes.splitlines() if "rust-lang/rust" in line) + rust_lang_remote = rust_lang_remote.split()[0] + + # Find which commit to compare to + merge_base = ["git", "merge-base", "HEAD", "{}/master".format(rust_lang_remote)] + commit = subprocess.check_output(merge_base, universal_newlines=True).strip() + + # Warn if there were changes to the compiler since the ancestor commit. + rev_parse = ["git", "rev-parse", "--show-toplevel"] + top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip() + compiler = "{}/compiler/".format(top_level) + status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler]) + if status != 0: + print("warning: `download-rustc` is enabled, but there are changes to compiler/") + + return commit + def rustc_stamp(self): """Return the path for .rustc-stamp @@ -1090,6 +1149,13 @@ def bootstrap(help_triggered): build.update_submodules() # Fetch/build the bootstrap + build.rustc_commit = build.maybe_download_rustc() + if build.rustc_commit is not None: + if build.verbose: + commit = build.rustc_commit + print("using downloaded stage1 artifacts from CI (commit {})".format(commit)) + # FIXME: support downloading artifacts from the beta channel + build.rustc_channel = "nightly" build.download_stage0() sys.stdout.flush() build.ensure_vendored() diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index f1a160250db..6ab106f3024 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -57,6 +57,14 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { /// `true` here can still be overwritten by `should_run` calling `default_condition`. const DEFAULT: bool = false; + /// Whether this step should be run even when `download-rustc` is set. + /// + /// Most steps are not important when the compiler is downloaded, since they will be included in + /// the pre-compiled sysroot. Steps can set this to `true` to be built anyway. + /// + /// When in doubt, set this to `false`. + const ENABLE_DOWNLOAD_RUSTC: bool = false; + /// If true, then this rule should be skipped if --target was specified, but --host was not const ONLY_HOSTS: bool = false; @@ -99,6 +107,7 @@ impl RunConfig<'_> { struct StepDescription { default: bool, + enable_download_rustc: bool, only_hosts: bool, should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, make_run: fn(RunConfig<'_>), @@ -153,6 +162,7 @@ impl StepDescription { fn from() -> StepDescription { StepDescription { default: S::DEFAULT, + enable_download_rustc: S::ENABLE_DOWNLOAD_RUSTC, only_hosts: S::ONLY_HOSTS, should_run: S::should_run, make_run: S::make_run, @@ -169,6 +179,14 @@ impl StepDescription { "{:?} not skipped for {:?} -- not in {:?}", pathset, self.name, builder.config.exclude ); + } else if builder.config.download_rustc && !self.enable_download_rustc { + if !builder.config.dry_run { + eprintln!( + "Not running {} because its artifacts have been downloaded from CI (`download-rustc` is set)", + self.name + ); + } + return; } // Determine the targets participating in this rule. @@ -629,8 +647,12 @@ impl<'a> Builder<'a> { .join("rustlib") .join(self.target.triple) .join("lib"); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); + // Avoid deleting the rustlib/ directory we just copied + // (in `impl Step for Sysroot`). + if !builder.config.download_rustc { + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + } INTERNER.intern_path(sysroot) } } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 6626fead774..9b80f1cf9fc 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -62,6 +62,7 @@ fn cargo_subcommand(kind: Kind) -> &'static str { impl Step for Std { type Output = (); const DEFAULT: bool = true; + const ENABLE_DOWNLOAD_RUSTC: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.all_krates("test") @@ -155,6 +156,7 @@ impl Step for Rustc { type Output = (); const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; + const ENABLE_DOWNLOAD_RUSTC: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.all_krates("rustc-main") @@ -233,6 +235,7 @@ impl Step for CodegenBackend { type Output = (); const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; + const ENABLE_DOWNLOAD_RUSTC: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"]) @@ -290,6 +293,7 @@ macro_rules! tool_check_step { type Output = (); const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; + const ENABLE_DOWNLOAD_RUSTC: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path($path) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 34002019a6f..833c13e9a26 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -41,7 +41,10 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.all_krates("test") + // When downloading stage1, the standard library has already been copied to the sysroot, so + // there's no need to rebuild it. + let download_rustc = run.builder.config.download_rustc; + run.all_krates("test").default_condition(!download_rustc) } fn make_run(run: RunConfig<'_>) { @@ -904,6 +907,18 @@ impl Step for Sysroot { let _ = fs::remove_dir_all(&sysroot); t!(fs::create_dir_all(&sysroot)); + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.config.download_rustc { + assert_eq!( + builder.config.build, compiler.host, + "Cross-compiling is not yet supported with `download-rustc`", + ); + // Copy the compiler into the correct sysroot. + let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0"); + builder.cp_r(&stage0_dir, &sysroot); + return INTERNER.intern_path(sysroot); + } + // Symlink the source root into the same location inside the sysroot, // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), // so that any tools relying on `rust-src` also work for local builds, @@ -975,13 +990,16 @@ impl Step for Assemble { // produce some other architecture compiler we need to start from // `build` to get there. // - // FIXME: Perhaps we should download those libraries? - // It would make builds faster... - // // FIXME: It may be faster if we build just a stage 1 compiler and then // use that to bootstrap this compiler forward. let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.config.download_rustc { + builder.ensure(Sysroot { compiler: target_compiler }); + return target_compiler; + } + // Build the libraries for this compiler to link to (i.e., the libraries // it uses at runtime). NOTE: Crates the target compiler compiles don't // link to these. (FIXME: Is that correct? It seems to be correct most diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index ec1308ab82b..e4b8269f94c 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -80,6 +80,7 @@ pub struct Config { pub cmd: Subcommand, pub incremental: bool, pub dry_run: bool, + pub download_rustc: bool, pub deny_warnings: bool, pub backtrace_on_ice: bool, @@ -503,6 +504,7 @@ struct Rust { new_symbol_mangling: Option, profile_generate: Option, profile_use: Option, + download_rustc: Option, } /// TOML representation of how each build target is configured. @@ -885,6 +887,7 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + config.download_rustc = rust.download_rustc.unwrap_or(false); } else { config.rust_profile_use = flags.rust_profile_use; config.rust_profile_generate = flags.rust_profile_generate; diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index bf6bea539e5..5c874f69bd9 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -477,6 +477,7 @@ pub struct Rustdoc { impl Step for Rustdoc { type Output = PathBuf; const DEFAULT: bool = true; + const ENABLE_DOWNLOAD_RUSTC: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {