OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Absolute bare minimum for downloading rustc from CI

Browse Source 
- Use the same compiler for stage0 and stage1. This should be fixed at
  some point (so bootstrap isn't constantly rebuilt).
- Make sure `x.py build` and `x.py check` work.
- Use `git merge-base` to determine the most recent commit to download.
- Copy stage0 to the various sysroots in `Sysroot`, and delegate to
  Sysroot in Assemble. Leave all other code unchanged.
- Rename date -> key

  This can also be a commit hash, so 'date' is no longer a good name.

- Add the commented-out option to config.toml.example
- Disable all steps by default when `download-rustc` is enabled

  Most steps don't make sense when downloading a compiler, because they'll
  be pre-built in the sysroot. Only enable the ones that might be useful,
  in particular Rustdoc and all `check` steps.

  At some point, this should probably enable other tools, but rustdoc is
  enough to test out `download-rustc`.

- Don't print 'Skipping' twice in a row

  Bootstrap forcibly enables a dry run if it isn't already set, so
  previously it would print the message twice:

  ```
  Skipping bootstrap::compile::Std because it is not enabled for `download-rustc`
  Skipping bootstrap::compile::Std because it is not enabled for `download-rustc`
  ```

  Now it correctly only prints once.

 ## Future work

- Add FIXME about supporting beta commits
- Debug logging will never work. This should be fixed.
This commit is contained in:
Joshua Nelson 2021-01-22 00:31:17 -05:00
parent 9a1d6174c9
commit 6457845219
7 changed files with 138 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.toml.example
View File

@ -358,6 +358,12 @@ changelog-seen = 2
#
#debug = false
# Whether to download the stage 1 and 2 compilers from CI.
# This is mostly useful for tools; if you have changes to `compiler/` they will be ignored.
#
# FIXME: currently, this also uses the downloaded compiler for stage0, but that causes unnecessary rebuilds.
#download-rustc = false
# Number of codegen units to use for each compiler invocation. A value of 0
# means "the number of cores on this machine", and 1+ is passed through to the
# compiler.

90
src/bootstrap/bootstrap.py
View File

@ -378,6 +378,7 @@ class RustBuild(object):
self.verbose = False
self.git_version = None
self.nix_deps_dir = None
self.rustc_commit = None
def download_stage0(self):
"""Fetch the build system for Rust, written in Rust
@ -394,20 +395,27 @@ class RustBuild(object):
if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp(), self.date)):
self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
download_rustc = self.rustc_commit is not None
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
self._download_stage0_helper(filename, pattern, tarball_suffix)
self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_stage0_helper(filename, "rustc", tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_stage0_helper(filename, "cargo", tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
if self.rustc_commit is not None:
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
self._download_component_helper(
filename, "rustc-dev", tarball_suffix, download_rustc
)
self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
@ -416,7 +424,7 @@ class RustBuild(object):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(self.date)
rust_stamp.write(self.date + str(self.rustc_commit))
if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
not os.path.exists(self.rustfmt())
@ -426,7 +434,9 @@ class RustBuild(object):
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
[channel, date] = rustfmt_channel.split('-', 1)
filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
self._download_component_helper(
filename, "rustfmt-preview", tarball_suffix, key=date
)
self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
@ -482,18 +492,27 @@ class RustBuild(object):
return opt == "true" \
or (opt == "if-available" and self.build in supported_platforms)
def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
if date is None:
date = self.date
def _download_component_helper(
self, filename, pattern, tarball_suffix, download_rustc=False, key=None
):
if key is None:
if download_rustc:
key = self.rustc_commit
else:
key = self.date
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, date)
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
url = "{}/dist/{}".format(self._download_url, date)
if download_rustc:
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
else:
url = "{}/dist/{}".format(self._download_url, key)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
do_verify = not download_rustc
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
@ -613,6 +632,46 @@ class RustBuild(object):
print("warning: failed to call patchelf:", reason)
return
# Return the stage1 compiler to download, if any.
def maybe_download_rustc(self):
# If `download-rustc` is not set, default to rebuilding.
if self.get_toml("download-rustc", section="rust") != "true":
return None
# Look for a version to compare to based on the current commit.
# There are a few different cases to handle.
# 1. This commit is a fast-forward from master: `master - * - * - HEAD`
# 2. This commit and master have diverged:
# ```
# Y - * - HEAD
# /
# X - * - master
# ```
# In this case, we should compare to `X`.
# 3. `master` and `HEAD` are radically different (>100 commits, or similar). This probably
# means that `master` does *not* correspond to the version we want to compare to, e.g. a
# fork. Instead, we want to compare to `rust-lang/rust:master`, which this has to share a
# recent merge base with.
# Find which remote corresponds to `rust-lang/rust`.
remotes = subprocess.check_output(["git", "remote", "-v"], universal_newlines=True)
# e.g. `origin https://github.com//rust-lang/rust (fetch)`
rust_lang_remote = next(line for line in remotes.splitlines() if "rust-lang/rust" in line)
rust_lang_remote = rust_lang_remote.split()[0]
# Find which commit to compare to
merge_base = ["git", "merge-base", "HEAD", "{}/master".format(rust_lang_remote)]
commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
# Warn if there were changes to the compiler since the ancestor commit.
rev_parse = ["git", "rev-parse", "--show-toplevel"]
top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
compiler = "{}/compiler/".format(top_level)
status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
if status != 0:
print("warning: `download-rustc` is enabled, but there are changes to compiler/")
return commit
def rustc_stamp(self):
"""Return the path for .rustc-stamp
@ -1090,6 +1149,13 @@ def bootstrap(help_triggered):
build.update_submodules()
# Fetch/build the bootstrap
build.rustc_commit = build.maybe_download_rustc()
if build.rustc_commit is not None:
if build.verbose:
commit = build.rustc_commit
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
# FIXME: support downloading artifacts from the beta channel
build.rustc_channel = "nightly"
build.download_stage0()
sys.stdout.flush()
build.ensure_vendored()

26
src/bootstrap/builder.rs
View File

@ -57,6 +57,14 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash {
/// `true` here can still be overwritten by `should_run` calling `default_condition`.
const DEFAULT: bool = false;
/// Whether this step should be run even when `download-rustc` is set.
///
/// Most steps are not important when the compiler is downloaded, since they will be included in
/// the pre-compiled sysroot. Steps can set this to `true` to be built anyway.
///
/// When in doubt, set this to `false`.
const ENABLE_DOWNLOAD_RUSTC: bool = false;
/// If true, then this rule should be skipped if --target was specified, but --host was not
const ONLY_HOSTS: bool = false;
@ -99,6 +107,7 @@ impl RunConfig<'_> {
struct StepDescription {
default: bool,
enable_download_rustc: bool,
only_hosts: bool,
should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>,
make_run: fn(RunConfig<'_>),
@ -153,6 +162,7 @@ impl StepDescription {
fn from<S: Step>() -> StepDescription {
StepDescription {
default: S::DEFAULT,
enable_download_rustc: S::ENABLE_DOWNLOAD_RUSTC,
only_hosts: S::ONLY_HOSTS,
should_run: S::should_run,
make_run: S::make_run,
@ -169,6 +179,14 @@ impl StepDescription {
"{:?} not skipped for {:?} -- not in {:?}",
pathset, self.name, builder.config.exclude
);
} else if builder.config.download_rustc && !self.enable_download_rustc {
if !builder.config.dry_run {
eprintln!(
"Not running {} because its artifacts have been downloaded from CI (`download-rustc` is set)",
self.name
);
}
return;
}
// Determine the targets participating in this rule.
@ -629,8 +647,12 @@ impl<'a> Builder<'a> {
.join("rustlib")
.join(self.target.triple)
.join("lib");
let _ = fs::remove_dir_all(&sysroot);
t!(fs::create_dir_all(&sysroot));
// Avoid deleting the rustlib/ directory we just copied
// (in `impl Step for Sysroot`).
if !builder.config.download_rustc {
let _ = fs::remove_dir_all(&sysroot);
t!(fs::create_dir_all(&sysroot));
}
INTERNER.intern_path(sysroot)
}
}

4
src/bootstrap/check.rs
View File

@ -62,6 +62,7 @@ fn cargo_subcommand(kind: Kind) -> &'static str {
impl Step for Std {
type Output = ();
const DEFAULT: bool = true;
const ENABLE_DOWNLOAD_RUSTC: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.all_krates("test")
@ -155,6 +156,7 @@ impl Step for Rustc {
type Output = ();
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = true;
const ENABLE_DOWNLOAD_RUSTC: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.all_krates("rustc-main")
@ -233,6 +235,7 @@ impl Step for CodegenBackend {
type Output = ();
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = true;
const ENABLE_DOWNLOAD_RUSTC: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"])
@ -290,6 +293,7 @@ macro_rules! tool_check_step {
type Output = ();
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = true;
const ENABLE_DOWNLOAD_RUSTC: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path($path)

26
src/bootstrap/compile.rs
View File

@ -41,7 +41,10 @@ impl Step for Std {
const DEFAULT: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.all_krates("test")
// When downloading stage1, the standard library has already been copied to the sysroot, so
// there's no need to rebuild it.
let download_rustc = run.builder.config.download_rustc;
run.all_krates("test").default_condition(!download_rustc)
}
fn make_run(run: RunConfig<'_>) {
@ -904,6 +907,18 @@ impl Step for Sysroot {
let _ = fs::remove_dir_all(&sysroot);
t!(fs::create_dir_all(&sysroot));
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc {
assert_eq!(
builder.config.build, compiler.host,
"Cross-compiling is not yet supported with `download-rustc`",
);
// Copy the compiler into the correct sysroot.
let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0");
builder.cp_r(&stage0_dir, &sysroot);
return INTERNER.intern_path(sysroot);
}
// Symlink the source root into the same location inside the sysroot,
// where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`),
// so that any tools relying on `rust-src` also work for local builds,
@ -975,13 +990,16 @@ impl Step for Assemble {
// produce some other architecture compiler we need to start from
// `build` to get there.
//
// FIXME: Perhaps we should download those libraries?
// It would make builds faster...
//
// FIXME: It may be faster if we build just a stage 1 compiler and then
// use that to bootstrap this compiler forward.
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc {
builder.ensure(Sysroot { compiler: target_compiler });
return target_compiler;
}
// Build the libraries for this compiler to link to (i.e., the libraries
// it uses at runtime). NOTE: Crates the target compiler compiles don't
// link to these. (FIXME: Is that correct? It seems to be correct most

3
src/bootstrap/config.rs
View File

@ -80,6 +80,7 @@ pub struct Config {
pub cmd: Subcommand,
pub incremental: bool,
pub dry_run: bool,
pub download_rustc: bool,
pub deny_warnings: bool,
pub backtrace_on_ice: bool,
@ -503,6 +504,7 @@ struct Rust {
new_symbol_mangling: Option<bool>,
profile_generate: Option<String>,
profile_use: Option<String>,
download_rustc: Option<bool>,
}
/// TOML representation of how each build target is configured.
@ -885,6 +887,7 @@ impl Config {
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc = rust.download_rustc.unwrap_or(false);
} else {
config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate;

1
src/bootstrap/tool.rs
View File

@ -477,6 +477,7 @@ pub struct Rustdoc {
impl Step for Rustdoc {
type Output = PathBuf;
const DEFAULT: bool = true;
const ENABLE_DOWNLOAD_RUSTC: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {