Absolute bare minimum for downloading rustc from CI
- Use the same compiler for stage0 and stage1. This should be fixed at some point (so bootstrap isn't constantly rebuilt). - Make sure `x.py build` and `x.py check` work. - Use `git merge-base` to determine the most recent commit to download. - Copy stage0 to the various sysroots in `Sysroot`, and delegate to Sysroot in Assemble. Leave all other code unchanged. - Rename date -> key This can also be a commit hash, so 'date' is no longer a good name. - Add the commented-out option to config.toml.example - Disable all steps by default when `download-rustc` is enabled Most steps don't make sense when downloading a compiler, because they'll be pre-built in the sysroot. Only enable the ones that might be useful, in particular Rustdoc and all `check` steps. At some point, this should probably enable other tools, but rustdoc is enough to test out `download-rustc`. - Don't print 'Skipping' twice in a row Bootstrap forcibly enables a dry run if it isn't already set, so previously it would print the message twice: ``` Skipping bootstrap::compile::Std because it is not enabled for `download-rustc` Skipping bootstrap::compile::Std because it is not enabled for `download-rustc` ``` Now it correctly only prints once. ## Future work - Add FIXME about supporting beta commits - Debug logging will never work. This should be fixed.
This commit is contained in:
Joshua Nelson
parent
9a1d6174c9
commit
6457845219
|
|
@ -358,6 +358,12 @@ changelog-seen = 2
|
|||
#
|
||||
#debug = false
|
||||
|
||||
# Whether to download the stage 1 and 2 compilers from CI.
|
||||
# This is mostly useful for tools; if you have changes to `compiler/` they will be ignored.
|
||||
#
|
||||
# FIXME: currently, this also uses the downloaded compiler for stage0, but that causes unnecessary rebuilds.
|
||||
#download-rustc = false
|
||||
|
||||
# Number of codegen units to use for each compiler invocation. A value of 0
|
||||
# means "the number of cores on this machine", and 1+ is passed through to the
|
||||
# compiler.
|
||||
|
|
|
|||
|
|
@ -378,6 +378,7 @@ class RustBuild(object):
|
|||
self.verbose = False
|
||||
self.git_version = None
|
||||
self.nix_deps_dir = None
|
||||
self.rustc_commit = None
|
||||
|
||||
def download_stage0(self):
|
||||
"""Fetch the build system for Rust, written in Rust
|
||||
|
|
@ -394,20 +395,27 @@ class RustBuild(object):
|
|||
|
||||
if self.rustc().startswith(self.bin_root()) and \
|
||||
(not os.path.exists(self.rustc()) or
|
||||
self.program_out_of_date(self.rustc_stamp(), self.date)):
|
||||
self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
|
||||
if os.path.exists(self.bin_root()):
|
||||
shutil.rmtree(self.bin_root())
|
||||
download_rustc = self.rustc_commit is not None
|
||||
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
|
||||
filename = "rust-std-{}-{}{}".format(
|
||||
rustc_channel, self.build, tarball_suffix)
|
||||
pattern = "rust-std-{}".format(self.build)
|
||||
self._download_stage0_helper(filename, pattern, tarball_suffix)
|
||||
self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
|
||||
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
|
||||
tarball_suffix)
|
||||
self._download_stage0_helper(filename, "rustc", tarball_suffix)
|
||||
self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
|
||||
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
|
||||
tarball_suffix)
|
||||
self._download_stage0_helper(filename, "cargo", tarball_suffix)
|
||||
self._download_component_helper(filename, "cargo", tarball_suffix)
|
||||
if self.rustc_commit is not None:
|
||||
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
|
||||
self._download_component_helper(
|
||||
filename, "rustc-dev", tarball_suffix, download_rustc
|
||||
)
|
||||
|
||||
self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
|
||||
self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
|
||||
self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
|
||||
|
|
@ -416,7 +424,7 @@ class RustBuild(object):
|
|||
if lib.endswith(".so"):
|
||||
self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
|
||||
with output(self.rustc_stamp()) as rust_stamp:
|
||||
rust_stamp.write(self.date)
|
||||
rust_stamp.write(self.date + str(self.rustc_commit))
|
||||
|
||||
if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
|
||||
not os.path.exists(self.rustfmt())
|
||||
|
|
@ -426,7 +434,9 @@ class RustBuild(object):
|
|||
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
|
||||
[channel, date] = rustfmt_channel.split('-', 1)
|
||||
filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
|
||||
self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
|
||||
self._download_component_helper(
|
||||
filename, "rustfmt-preview", tarball_suffix, key=date
|
||||
)
|
||||
self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
|
||||
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
|
||||
with output(self.rustfmt_stamp()) as rustfmt_stamp:
|
||||
|
|
@ -482,18 +492,27 @@ class RustBuild(object):
|
|||
return opt == "true" \
|
||||
or (opt == "if-available" and self.build in supported_platforms)
|
||||
|
||||
def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
|
||||
if date is None:
|
||||
date = self.date
|
||||
def _download_component_helper(
|
||||
self, filename, pattern, tarball_suffix, download_rustc=False, key=None
|
||||
):
|
||||
if key is None:
|
||||
if download_rustc:
|
||||
key = self.rustc_commit
|
||||
else:
|
||||
key = self.date
|
||||
cache_dst = os.path.join(self.build_dir, "cache")
|
||||
rustc_cache = os.path.join(cache_dst, date)
|
||||
rustc_cache = os.path.join(cache_dst, key)
|
||||
if not os.path.exists(rustc_cache):
|
||||
os.makedirs(rustc_cache)
|
||||
|
||||
url = "{}/dist/{}".format(self._download_url, date)
|
||||
if download_rustc:
|
||||
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
|
||||
else:
|
||||
url = "{}/dist/{}".format(self._download_url, key)
|
||||
tarball = os.path.join(rustc_cache, filename)
|
||||
if not os.path.exists(tarball):
|
||||
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
|
||||
do_verify = not download_rustc
|
||||
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
|
||||
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
|
||||
|
||||
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
|
||||
|
|
@ -613,6 +632,46 @@ class RustBuild(object):
|
|||
print("warning: failed to call patchelf:", reason)
|
||||
return
|
||||
|
||||
# Return the stage1 compiler to download, if any.
|
||||
def maybe_download_rustc(self):
|
||||
# If `download-rustc` is not set, default to rebuilding.
|
||||
if self.get_toml("download-rustc", section="rust") != "true":
|
||||
return None
|
||||
# Look for a version to compare to based on the current commit.
|
||||
# There are a few different cases to handle.
|
||||
# 1. This commit is a fast-forward from master: `master - * - * - HEAD`
|
||||
# 2. This commit and master have diverged:
|
||||
# ```
|
||||
# Y - * - HEAD
|
||||
# /
|
||||
# X - * - master
|
||||
# ```
|
||||
# In this case, we should compare to `X`.
|
||||
# 3. `master` and `HEAD` are radically different (>100 commits, or similar). This probably
|
||||
# means that `master` does *not* correspond to the version we want to compare to, e.g. a
|
||||
# fork. Instead, we want to compare to `rust-lang/rust:master`, which this has to share a
|
||||
# recent merge base with.
|
||||
|
||||
# Find which remote corresponds to `rust-lang/rust`.
|
||||
remotes = subprocess.check_output(["git", "remote", "-v"], universal_newlines=True)
|
||||
# e.g. `origin https://github.com//rust-lang/rust (fetch)`
|
||||
rust_lang_remote = next(line for line in remotes.splitlines() if "rust-lang/rust" in line)
|
||||
rust_lang_remote = rust_lang_remote.split()[0]
|
||||
|
||||
# Find which commit to compare to
|
||||
merge_base = ["git", "merge-base", "HEAD", "{}/master".format(rust_lang_remote)]
|
||||
commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
|
||||
|
||||
# Warn if there were changes to the compiler since the ancestor commit.
|
||||
rev_parse = ["git", "rev-parse", "--show-toplevel"]
|
||||
top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
|
||||
compiler = "{}/compiler/".format(top_level)
|
||||
status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
|
||||
if status != 0:
|
||||
print("warning: `download-rustc` is enabled, but there are changes to compiler/")
|
||||
|
||||
return commit
|
||||
|
||||
def rustc_stamp(self):
|
||||
"""Return the path for .rustc-stamp
|
||||
|
||||
|
|
@ -1090,6 +1149,13 @@ def bootstrap(help_triggered):
|
|||
build.update_submodules()
|
||||
|
||||
# Fetch/build the bootstrap
|
||||
build.rustc_commit = build.maybe_download_rustc()
|
||||
if build.rustc_commit is not None:
|
||||
if build.verbose:
|
||||
commit = build.rustc_commit
|
||||
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
|
||||
# FIXME: support downloading artifacts from the beta channel
|
||||
build.rustc_channel = "nightly"
|
||||
build.download_stage0()
|
||||
sys.stdout.flush()
|
||||
build.ensure_vendored()
|
||||
|
|
|
|||
|
|
@ -57,6 +57,14 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash {
|
|||
/// `true` here can still be overwritten by `should_run` calling `default_condition`.
|
||||
const DEFAULT: bool = false;
|
||||
|
||||
/// Whether this step should be run even when `download-rustc` is set.
|
||||
///
|
||||
/// Most steps are not important when the compiler is downloaded, since they will be included in
|
||||
/// the pre-compiled sysroot. Steps can set this to `true` to be built anyway.
|
||||
///
|
||||
/// When in doubt, set this to `false`.
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = false;
|
||||
|
||||
/// If true, then this rule should be skipped if --target was specified, but --host was not
|
||||
const ONLY_HOSTS: bool = false;
|
||||
|
||||
|
|
@ -99,6 +107,7 @@ impl RunConfig<'_> {
|
|||
|
||||
struct StepDescription {
|
||||
default: bool,
|
||||
enable_download_rustc: bool,
|
||||
only_hosts: bool,
|
||||
should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>,
|
||||
make_run: fn(RunConfig<'_>),
|
||||
|
|
@ -153,6 +162,7 @@ impl StepDescription {
|
|||
fn from<S: Step>() -> StepDescription {
|
||||
StepDescription {
|
||||
default: S::DEFAULT,
|
||||
enable_download_rustc: S::ENABLE_DOWNLOAD_RUSTC,
|
||||
only_hosts: S::ONLY_HOSTS,
|
||||
should_run: S::should_run,
|
||||
make_run: S::make_run,
|
||||
|
|
@ -169,6 +179,14 @@ impl StepDescription {
|
|||
"{:?} not skipped for {:?} -- not in {:?}",
|
||||
pathset, self.name, builder.config.exclude
|
||||
);
|
||||
} else if builder.config.download_rustc && !self.enable_download_rustc {
|
||||
if !builder.config.dry_run {
|
||||
eprintln!(
|
||||
"Not running {} because its artifacts have been downloaded from CI (`download-rustc` is set)",
|
||||
self.name
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine the targets participating in this rule.
|
||||
|
|
@ -629,8 +647,12 @@ impl<'a> Builder<'a> {
|
|||
.join("rustlib")
|
||||
.join(self.target.triple)
|
||||
.join("lib");
|
||||
let _ = fs::remove_dir_all(&sysroot);
|
||||
t!(fs::create_dir_all(&sysroot));
|
||||
// Avoid deleting the rustlib/ directory we just copied
|
||||
// (in `impl Step for Sysroot`).
|
||||
if !builder.config.download_rustc {
|
||||
let _ = fs::remove_dir_all(&sysroot);
|
||||
t!(fs::create_dir_all(&sysroot));
|
||||
}
|
||||
INTERNER.intern_path(sysroot)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ fn cargo_subcommand(kind: Kind) -> &'static str {
|
|||
impl Step for Std {
|
||||
type Output = ();
|
||||
const DEFAULT: bool = true;
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
run.all_krates("test")
|
||||
|
|
@ -155,6 +156,7 @@ impl Step for Rustc {
|
|||
type Output = ();
|
||||
const ONLY_HOSTS: bool = true;
|
||||
const DEFAULT: bool = true;
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
run.all_krates("rustc-main")
|
||||
|
|
@ -233,6 +235,7 @@ impl Step for CodegenBackend {
|
|||
type Output = ();
|
||||
const ONLY_HOSTS: bool = true;
|
||||
const DEFAULT: bool = true;
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"])
|
||||
|
|
@ -290,6 +293,7 @@ macro_rules! tool_check_step {
|
|||
type Output = ();
|
||||
const ONLY_HOSTS: bool = true;
|
||||
const DEFAULT: bool = true;
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
run.path($path)
|
||||
|
|
|
|||
|
|
@ -41,7 +41,10 @@ impl Step for Std {
|
|||
const DEFAULT: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
run.all_krates("test")
|
||||
// When downloading stage1, the standard library has already been copied to the sysroot, so
|
||||
// there's no need to rebuild it.
|
||||
let download_rustc = run.builder.config.download_rustc;
|
||||
run.all_krates("test").default_condition(!download_rustc)
|
||||
}
|
||||
|
||||
fn make_run(run: RunConfig<'_>) {
|
||||
|
|
@ -904,6 +907,18 @@ impl Step for Sysroot {
|
|||
let _ = fs::remove_dir_all(&sysroot);
|
||||
t!(fs::create_dir_all(&sysroot));
|
||||
|
||||
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
||||
if builder.config.download_rustc {
|
||||
assert_eq!(
|
||||
builder.config.build, compiler.host,
|
||||
"Cross-compiling is not yet supported with `download-rustc`",
|
||||
);
|
||||
// Copy the compiler into the correct sysroot.
|
||||
let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0");
|
||||
builder.cp_r(&stage0_dir, &sysroot);
|
||||
return INTERNER.intern_path(sysroot);
|
||||
}
|
||||
|
||||
// Symlink the source root into the same location inside the sysroot,
|
||||
// where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`),
|
||||
// so that any tools relying on `rust-src` also work for local builds,
|
||||
|
|
@ -975,13 +990,16 @@ impl Step for Assemble {
|
|||
// produce some other architecture compiler we need to start from
|
||||
// `build` to get there.
|
||||
//
|
||||
// FIXME: Perhaps we should download those libraries?
|
||||
// It would make builds faster...
|
||||
//
|
||||
// FIXME: It may be faster if we build just a stage 1 compiler and then
|
||||
// use that to bootstrap this compiler forward.
|
||||
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
|
||||
|
||||
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
||||
if builder.config.download_rustc {
|
||||
builder.ensure(Sysroot { compiler: target_compiler });
|
||||
return target_compiler;
|
||||
}
|
||||
|
||||
// Build the libraries for this compiler to link to (i.e., the libraries
|
||||
// it uses at runtime). NOTE: Crates the target compiler compiles don't
|
||||
// link to these. (FIXME: Is that correct? It seems to be correct most
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ pub struct Config {
|
|||
pub cmd: Subcommand,
|
||||
pub incremental: bool,
|
||||
pub dry_run: bool,
|
||||
pub download_rustc: bool,
|
||||
|
||||
pub deny_warnings: bool,
|
||||
pub backtrace_on_ice: bool,
|
||||
|
|
@ -503,6 +504,7 @@ struct Rust {
|
|||
new_symbol_mangling: Option<bool>,
|
||||
profile_generate: Option<String>,
|
||||
profile_use: Option<String>,
|
||||
download_rustc: Option<bool>,
|
||||
}
|
||||
|
||||
/// TOML representation of how each build target is configured.
|
||||
|
|
@ -885,6 +887,7 @@ impl Config {
|
|||
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
|
||||
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
|
||||
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
|
||||
config.download_rustc = rust.download_rustc.unwrap_or(false);
|
||||
} else {
|
||||
config.rust_profile_use = flags.rust_profile_use;
|
||||
config.rust_profile_generate = flags.rust_profile_generate;
|
||||
|
|
|
|||
|
|
@ -477,6 +477,7 @@ pub struct Rustdoc {
|
|||
impl Step for Rustdoc {
|
||||
type Output = PathBuf;
|
||||
const DEFAULT: bool = true;
|
||||
const ENABLE_DOWNLOAD_RUSTC: bool = true;
|
||||
const ONLY_HOSTS: bool = true;
|
||||
|
||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||
|
|
|
|||
Loading…
Reference in New Issue