From a7b092f4188e927b3456fe6fe5ee2bd72c53cb57 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Fri, 4 Sep 2020 19:00:04 -0400 Subject: [PATCH] Download LLVM from CI to bootstrap --- config.toml.example | 15 ++++++ src/bootstrap/bootstrap.py | 105 +++++++++++++++++++++++++++++-------- src/bootstrap/config.rs | 57 ++++++++++++++++++++ src/bootstrap/dist.rs | 34 +++++++----- src/bootstrap/lib.rs | 4 ++ 5 files changed, 178 insertions(+), 37 deletions(-) diff --git a/config.toml.example b/config.toml.example index 2d5b3136450..b50406a9579 100644 --- a/config.toml.example +++ b/config.toml.example @@ -14,6 +14,21 @@ # ============================================================================= [llvm] +# Whether to use Rust CI built LLVM instead of locally building it. +# +# Unless you're developing for a target where Rust CI doesn't build a compiler +# toolchain or changing LLVM locally, you probably want to set this to true. +# +# It's currently false by default due to being newly added; please file bugs if +# enabling this did not work for you on Linux (macOS and Windows support is +# coming soon). +# +# We also currently only support this when building LLVM for the build triple. +# +# Note that many of the LLVM options are not currently supported for +# downloading. Currently only the "assertions" option can be toggled. +#download-ci-llvm = false + # Indicates whether LLVM rebuild should be skipped when running bootstrap. If # this is `false` then the compiler's LLVM will be rebuilt whenever the built # version doesn't have the correct hash. If it is `true` then LLVM will never diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 955c72856dc..44a17f75451 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -14,8 +14,17 @@ import tempfile from time import time +def support_xz(): + try: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tarfile.open(temp_path, "w:xz"): + pass + return True + except tarfile.CompressionError: + return False -def get(url, path, verbose=False): +def get(url, path, verbose=False, do_verify=True): suffix = '.sha256' sha_url = url + suffix with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -24,19 +33,20 @@ def get(url, path, verbose=False): sha_path = sha_file.name try: - download(sha_path, sha_url, False, verbose) - if os.path.exists(path): - if verify(path, sha_path, False): - if verbose: - print("using already-download file", path) - return - else: - if verbose: - print("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) + if do_verify: + download(sha_path, sha_url, False, verbose) + if os.path.exists(path): + if verify(path, sha_path, False): + if verbose: + print("using already-download file", path) + return + else: + if verbose: + print("ignoring already-download file", + path, "due to failed verification") + os.unlink(path) download(temp_path, url, True, verbose) - if not verify(temp_path, sha_path, verbose): + if do_verify and not verify(temp_path, sha_path, verbose): raise RuntimeError("failed verification") if verbose: print("moving {} to {}".format(temp_path, path)) @@ -365,16 +375,6 @@ class RustBuild(object): cargo_channel = self.cargo_channel rustfmt_channel = self.rustfmt_channel - def support_xz(): - try: - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - with tarfile.open(temp_path, "w:xz"): - pass - return True - except tarfile.CompressionError: - return False - if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.program_out_of_date(self.rustc_stamp())): @@ -423,6 +423,19 @@ class RustBuild(object): with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.date + self.rustfmt_channel) + if self.downloading_llvm(): + llvm_sha = subprocess.check_output(["git", "log", "--author=bors", + "--format=%H", "-n1"]).decode(sys.getdefaultencoding()).strip() + llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' + if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): + self._download_ci_llvm(llvm_sha, llvm_assertions) + with output(self.llvm_stamp()) as llvm_stamp: + llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions)) + + def downloading_llvm(self): + opt = self.get_toml('download-ci-llvm', 'llvm') + return opt == "true" + def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None): if date is None: date = self.date @@ -437,6 +450,25 @@ class RustBuild(object): get("{}/{}".format(url, filename), tarball, verbose=self.verbose) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) + def _download_ci_llvm(self, llvm_sha, llvm_assertions): + cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) + cache_dst = os.path.join(self.build_dir, "cache") + rustc_cache = os.path.join(cache_dst, cache_prefix) + if not os.path.exists(rustc_cache): + os.makedirs(rustc_cache) + + url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha) + if llvm_assertions: + url = url.replace('rustc-builds', 'rustc-builds-alt') + tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' + filename = "rust-dev-nightly-" + self.build + tarball_suffix + tarball = os.path.join(rustc_cache, filename) + if not os.path.exists(tarball): + get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False) + unpack(tarball, tarball_suffix, self.llvm_root(), + match="rust-dev", + verbose=self.verbose) + def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, or the RPATH section, to fix the dynamic library search path @@ -558,6 +590,17 @@ class RustBuild(object): """ return os.path.join(self.bin_root(), '.rustfmt-stamp') + def llvm_stamp(self): + """Return the path for .rustfmt-stamp + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") + True + """ + return os.path.join(self.llvm_root(), '.llvm-stamp') + + def program_out_of_date(self, stamp_path, extra=""): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -581,6 +624,22 @@ class RustBuild(object): """ return os.path.join(self.build_dir, self.build, "stage0") + def llvm_root(self): + """Return the CI LLVM root directory + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.llvm_root() == os.path.join("build", "ci-llvm") + True + + When the 'build' property is given should be a nested directory: + + >>> rb.build = "devel" + >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") + True + """ + return os.path.join(self.build_dir, self.build, "ci-llvm") + def get_toml(self, key, section=None): """Returns the value of the given key in config.toml, otherwise returns None diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 5a79d3db5c9..d889d04fa6f 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -15,9 +15,20 @@ use std::process; use crate::cache::{Interned, INTERNER}; use crate::flags::Flags; pub use crate::flags::Subcommand; +use crate::util::exe; use build_helper::t; use serde::Deserialize; +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name) + ); + }; +} + /// Global configuration for the entire build and/or bootstrap. /// /// This structure is derived from a combination of both `config.toml` and @@ -84,6 +95,7 @@ pub struct Config { pub llvm_version_suffix: Option, pub llvm_use_linker: Option, pub llvm_allow_old_toolchain: Option, + pub llvm_from_ci: bool, pub use_lld: bool, pub lld_enabled: bool, @@ -344,6 +356,7 @@ struct Llvm { use_libcxx: Option, use_linker: Option, allow_old_toolchain: Option, + download_ci_llvm: Option, } #[derive(Deserialize, Default, Clone)] @@ -624,6 +637,36 @@ impl Config { set(&mut config.llvm_use_libcxx, llvm.use_libcxx); config.llvm_use_linker = llvm.use_linker.clone(); config.llvm_allow_old_toolchain = llvm.allow_old_toolchain; + config.llvm_from_ci = llvm.download_ci_llvm.unwrap_or(false); + + if config.llvm_from_ci { + // None of the LLVM options, except assertions, are supported + // when using downloaded LLVM. We could just ignore these but + // that's potentially confusing, so force them to not be + // explicitly set. The defaults and CI defaults don't + // necessarily match but forcing people to match (somewhat + // arbitrary) CI configuration locally seems bad/hard. + check_ci_llvm!(llvm.optimize); + check_ci_llvm!(llvm.thin_lto); + check_ci_llvm!(llvm.release_debuginfo); + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.static_libstdcpp); + check_ci_llvm!(llvm.targets); + check_ci_llvm!(llvm.experimental_targets); + check_ci_llvm!(llvm.link_jobs); + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.clang_cl); + check_ci_llvm!(llvm.version_suffix); + check_ci_llvm!(llvm.cflags); + check_ci_llvm!(llvm.cxxflags); + check_ci_llvm!(llvm.ldflags); + check_ci_llvm!(llvm.use_libcxx); + check_ci_llvm!(llvm.use_linker); + check_ci_llvm!(llvm.allow_old_toolchain); + + // CI-built LLVM is shared + config.llvm_link_shared = true; + } } if let Some(ref rust) = toml.rust { @@ -706,6 +749,20 @@ impl Config { } } + if config.llvm_from_ci { + let triple = &config.build.triple; + let mut build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(&triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin"); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + if let Some(ref t) = toml.dist { config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e22cdb13928..cf73e570fa5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2382,26 +2382,32 @@ impl Step for HashSign { /// Note: This function does not yet support Windows, but we also don't support /// linking LLVM tools dynamically on Windows yet. fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) { - let src_libdir = builder.llvm_out(target).join("lib"); + if !builder.config.llvm_link_shared { + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + return; + } + // On macOS for some reason the llvm-config binary behaves differently and + // and fails on missing .a files if run without --link-shared. If run with + // that option, it still fails, but because we only ship a libLLVM.dylib + // rather than libLLVM-11-rust-....dylib file. + // + // For now just don't use llvm-config here on macOS; that will fail to + // support CI-built LLVM, but until we work out the different behavior that + // is fine as it is off by default. if target.contains("apple-darwin") { + let src_libdir = builder.llvm_out(target).join("lib"); let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); if llvm_dylib_path.exists() { builder.install(&llvm_dylib_path, dst_libdir, 0o644); } - return; - } - - // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so. - // Since tools link to the latter rather than the former, we have to - // follow the symlink to find out what to distribute. - let llvm_dylib_path = src_libdir.join("libLLVM.so"); - if llvm_dylib_path.exists() { - let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| { - panic!("dist: Error calling canonicalize path `{}`: {}", llvm_dylib_path.display(), e); - }); - - builder.install(&llvm_dylib_path, dst_libdir, 0o644); + } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) { + let files = output(Command::new(llvm_config).arg("--libfiles")); + for file in files.lines() { + builder.install(Path::new(file), dst_libdir, 0o644); + } } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index f0224d88226..170c93561ef 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -611,6 +611,10 @@ impl Build { /// /// If no custom `llvm-config` was specified then Rust's llvm will be used. fn is_rust_llvm(&self, target: TargetSelection) -> bool { + if self.config.llvm_from_ci && target == self.config.build { + return true; + } + match self.config.target_config.get(&target) { Some(ref c) => c.llvm_config.is_none(), None => true,