569 lines
19 KiB
Rust
569 lines
19 KiB
Rust
//! Build a dist manifest, hash and sign everything.
|
|
//! This gets called by `promote-release`
|
|
//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
|
|
//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
|
|
//! by rustbuild (in `src/bootstrap/dist.rs`).
|
|
|
|
mod checksum;
|
|
mod manifest;
|
|
mod versions;
|
|
|
|
use crate::checksum::Checksums;
|
|
use crate::manifest::{Component, Manifest, Package, Rename, Target};
|
|
use crate::versions::{PkgType, Versions};
|
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
|
use std::env;
|
|
use std::fs::{self, File};
|
|
use std::path::{Path, PathBuf};
|
|
|
|
static HOSTS: &[&str] = &[
|
|
"aarch64-apple-darwin",
|
|
"aarch64-pc-windows-msvc",
|
|
"aarch64-unknown-linux-gnu",
|
|
"aarch64-unknown-linux-musl",
|
|
"arm-unknown-linux-gnueabi",
|
|
"arm-unknown-linux-gnueabihf",
|
|
"armv7-unknown-linux-gnueabihf",
|
|
"i686-apple-darwin",
|
|
"i686-pc-windows-gnu",
|
|
"i686-pc-windows-msvc",
|
|
"i686-unknown-linux-gnu",
|
|
"mips-unknown-linux-gnu",
|
|
"mips64-unknown-linux-gnuabi64",
|
|
"mips64el-unknown-linux-gnuabi64",
|
|
"mipsel-unknown-linux-gnu",
|
|
"mipsisa32r6-unknown-linux-gnu",
|
|
"mipsisa32r6el-unknown-linux-gnu",
|
|
"mipsisa64r6-unknown-linux-gnuabi64",
|
|
"mipsisa64r6el-unknown-linux-gnuabi64",
|
|
"powerpc-unknown-linux-gnu",
|
|
"powerpc64-unknown-linux-gnu",
|
|
"powerpc64le-unknown-linux-gnu",
|
|
"riscv64gc-unknown-linux-gnu",
|
|
"s390x-unknown-linux-gnu",
|
|
"x86_64-apple-darwin",
|
|
"x86_64-pc-windows-gnu",
|
|
"x86_64-pc-windows-msvc",
|
|
"x86_64-unknown-freebsd",
|
|
"x86_64-unknown-illumos",
|
|
"x86_64-unknown-linux-gnu",
|
|
"x86_64-unknown-linux-musl",
|
|
"x86_64-unknown-netbsd",
|
|
];
|
|
|
|
static TARGETS: &[&str] = &[
|
|
"aarch64-apple-darwin",
|
|
"aarch64-apple-ios",
|
|
"aarch64-fuchsia",
|
|
"aarch64-linux-android",
|
|
"aarch64-pc-windows-msvc",
|
|
"aarch64-unknown-hermit",
|
|
"aarch64-unknown-linux-gnu",
|
|
"aarch64-unknown-linux-musl",
|
|
"aarch64-unknown-none",
|
|
"aarch64-unknown-none-softfloat",
|
|
"aarch64-unknown-redox",
|
|
"arm-linux-androideabi",
|
|
"arm-unknown-linux-gnueabi",
|
|
"arm-unknown-linux-gnueabihf",
|
|
"arm-unknown-linux-musleabi",
|
|
"arm-unknown-linux-musleabihf",
|
|
"armv5te-unknown-linux-gnueabi",
|
|
"armv5te-unknown-linux-musleabi",
|
|
"armv7-apple-ios",
|
|
"armv7-linux-androideabi",
|
|
"thumbv7neon-linux-androideabi",
|
|
"armv7-unknown-linux-gnueabi",
|
|
"armv7-unknown-linux-gnueabihf",
|
|
"armv7a-none-eabi",
|
|
"thumbv7neon-unknown-linux-gnueabihf",
|
|
"armv7-unknown-linux-musleabi",
|
|
"armv7-unknown-linux-musleabihf",
|
|
"armebv7r-none-eabi",
|
|
"armebv7r-none-eabihf",
|
|
"armv7r-none-eabi",
|
|
"armv7r-none-eabihf",
|
|
"armv7s-apple-ios",
|
|
"asmjs-unknown-emscripten",
|
|
"i386-apple-ios",
|
|
"i586-pc-windows-msvc",
|
|
"i586-unknown-linux-gnu",
|
|
"i586-unknown-linux-musl",
|
|
"i686-apple-darwin",
|
|
"i686-linux-android",
|
|
"i686-pc-windows-gnu",
|
|
"i686-pc-windows-msvc",
|
|
"i686-unknown-freebsd",
|
|
"i686-unknown-linux-gnu",
|
|
"i686-unknown-linux-musl",
|
|
"mips-unknown-linux-gnu",
|
|
"mips-unknown-linux-musl",
|
|
"mips64-unknown-linux-gnuabi64",
|
|
"mips64-unknown-linux-muslabi64",
|
|
"mips64el-unknown-linux-gnuabi64",
|
|
"mips64el-unknown-linux-muslabi64",
|
|
"mipsisa32r6-unknown-linux-gnu",
|
|
"mipsisa32r6el-unknown-linux-gnu",
|
|
"mipsisa64r6-unknown-linux-gnuabi64",
|
|
"mipsisa64r6el-unknown-linux-gnuabi64",
|
|
"mipsel-unknown-linux-gnu",
|
|
"mipsel-unknown-linux-musl",
|
|
"nvptx64-nvidia-cuda",
|
|
"powerpc-unknown-linux-gnu",
|
|
"powerpc64-unknown-linux-gnu",
|
|
"powerpc64le-unknown-linux-gnu",
|
|
"riscv32i-unknown-none-elf",
|
|
"riscv32imc-unknown-none-elf",
|
|
"riscv32imac-unknown-none-elf",
|
|
"riscv32gc-unknown-linux-gnu",
|
|
"riscv64imac-unknown-none-elf",
|
|
"riscv64gc-unknown-none-elf",
|
|
"riscv64gc-unknown-linux-gnu",
|
|
"s390x-unknown-linux-gnu",
|
|
"sparc64-unknown-linux-gnu",
|
|
"sparcv9-sun-solaris",
|
|
"thumbv6m-none-eabi",
|
|
"thumbv7em-none-eabi",
|
|
"thumbv7em-none-eabihf",
|
|
"thumbv7m-none-eabi",
|
|
"thumbv8m.base-none-eabi",
|
|
"thumbv8m.main-none-eabi",
|
|
"thumbv8m.main-none-eabihf",
|
|
"wasm32-unknown-emscripten",
|
|
"wasm32-unknown-unknown",
|
|
"wasm32-wasi",
|
|
"x86_64-apple-darwin",
|
|
"x86_64-apple-ios",
|
|
"x86_64-fortanix-unknown-sgx",
|
|
"x86_64-fuchsia",
|
|
"x86_64-linux-android",
|
|
"x86_64-pc-windows-gnu",
|
|
"x86_64-pc-windows-msvc",
|
|
"x86_64-rumprun-netbsd",
|
|
"x86_64-sun-solaris",
|
|
"x86_64-pc-solaris",
|
|
"x86_64-unknown-freebsd",
|
|
"x86_64-unknown-illumos",
|
|
"x86_64-unknown-linux-gnu",
|
|
"x86_64-unknown-linux-gnux32",
|
|
"x86_64-unknown-linux-musl",
|
|
"x86_64-unknown-netbsd",
|
|
"x86_64-unknown-redox",
|
|
"x86_64-unknown-hermit",
|
|
];
|
|
|
|
static DOCS_TARGETS: &[&str] = &[
|
|
"aarch64-unknown-linux-gnu",
|
|
"i686-apple-darwin",
|
|
"i686-pc-windows-gnu",
|
|
"i686-pc-windows-msvc",
|
|
"i686-unknown-linux-gnu",
|
|
"x86_64-apple-darwin",
|
|
"x86_64-pc-windows-gnu",
|
|
"x86_64-pc-windows-msvc",
|
|
"x86_64-unknown-linux-gnu",
|
|
"x86_64-unknown-linux-musl",
|
|
];
|
|
|
|
static MSI_INSTALLERS: &[&str] = &[
|
|
"aarch64-pc-windows-msvc",
|
|
"i686-pc-windows-gnu",
|
|
"i686-pc-windows-msvc",
|
|
"x86_64-pc-windows-gnu",
|
|
"x86_64-pc-windows-msvc",
|
|
];
|
|
|
|
static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
|
|
|
|
static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
|
|
|
|
static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
|
|
|
|
macro_rules! t {
|
|
($e:expr) => {
|
|
match $e {
|
|
Ok(e) => e,
|
|
Err(e) => panic!("{} failed with {}", stringify!($e), e),
|
|
}
|
|
};
|
|
}
|
|
|
|
struct Builder {
|
|
versions: Versions,
|
|
checksums: Checksums,
|
|
shipped_files: HashSet<String>,
|
|
|
|
input: PathBuf,
|
|
output: PathBuf,
|
|
s3_address: String,
|
|
date: String,
|
|
}
|
|
|
|
fn main() {
|
|
let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
|
|
num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
|
|
} else {
|
|
num_cpus::get()
|
|
};
|
|
rayon::ThreadPoolBuilder::new()
|
|
.num_threads(num_threads)
|
|
.build_global()
|
|
.expect("failed to initialize Rayon");
|
|
|
|
let mut args = env::args().skip(1);
|
|
let input = PathBuf::from(args.next().unwrap());
|
|
let output = PathBuf::from(args.next().unwrap());
|
|
let date = args.next().unwrap();
|
|
let s3_address = args.next().unwrap();
|
|
let channel = args.next().unwrap();
|
|
|
|
Builder {
|
|
versions: Versions::new(&channel, &input).unwrap(),
|
|
checksums: t!(Checksums::new()),
|
|
shipped_files: HashSet::new(),
|
|
|
|
input,
|
|
output,
|
|
s3_address,
|
|
date,
|
|
}
|
|
.build();
|
|
}
|
|
|
|
impl Builder {
|
|
fn build(&mut self) {
|
|
self.check_toolstate();
|
|
let manifest = self.build_manifest();
|
|
|
|
let channel = self.versions.channel().to_string();
|
|
self.write_channel_files(&channel, &manifest);
|
|
if channel == "stable" {
|
|
// channel-rust-1.XX.YY.toml
|
|
let rust_version = self.versions.rustc_version().to_string();
|
|
self.write_channel_files(&rust_version, &manifest);
|
|
|
|
// channel-rust-1.XX.toml
|
|
let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
|
|
self.write_channel_files(&major_minor, &manifest);
|
|
}
|
|
|
|
if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
|
|
self.write_shipped_files(&Path::new(&path));
|
|
}
|
|
|
|
t!(self.checksums.store_cache());
|
|
}
|
|
|
|
/// If a tool does not pass its tests, don't ship it.
|
|
/// Right now, we do this only for Miri.
|
|
fn check_toolstate(&mut self) {
|
|
for file in &["toolstates-linux.json", "toolstates-windows.json"] {
|
|
let toolstates: Option<HashMap<String, String>> = File::open(self.input.join(file))
|
|
.ok()
|
|
.and_then(|f| serde_json::from_reader(&f).ok());
|
|
let toolstates = toolstates.unwrap_or_else(|| {
|
|
println!("WARNING: `{}` missing/malformed; assuming all tools failed", file);
|
|
HashMap::default() // Use empty map if anything went wrong.
|
|
});
|
|
// Mark some tools as missing based on toolstate.
|
|
if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
|
|
println!("Miri tests are not passing, removing component");
|
|
self.versions.disable_version(&PkgType::Miri);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
fn build_manifest(&mut self) -> Manifest {
|
|
let mut manifest = Manifest {
|
|
manifest_version: "2".to_string(),
|
|
date: self.date.to_string(),
|
|
pkg: BTreeMap::new(),
|
|
artifacts: BTreeMap::new(),
|
|
renames: BTreeMap::new(),
|
|
profiles: BTreeMap::new(),
|
|
};
|
|
self.add_packages_to(&mut manifest);
|
|
self.add_artifacts_to(&mut manifest);
|
|
self.add_profiles_to(&mut manifest);
|
|
self.add_renames_to(&mut manifest);
|
|
manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
|
|
|
|
self.checksums.fill_missing_checksums(&mut manifest);
|
|
|
|
manifest
|
|
}
|
|
|
|
fn add_packages_to(&mut self, manifest: &mut Manifest) {
|
|
let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
|
|
package("rustc", HOSTS);
|
|
package("rustc-dev", HOSTS);
|
|
package("reproducible-artifacts", HOSTS);
|
|
package("rustc-docs", HOSTS);
|
|
package("cargo", HOSTS);
|
|
package("rust-mingw", MINGW);
|
|
package("rust-std", TARGETS);
|
|
package("rust-docs", DOCS_TARGETS);
|
|
package("rust-src", &["*"]);
|
|
package("rls-preview", HOSTS);
|
|
package("rust-analyzer-preview", HOSTS);
|
|
package("clippy-preview", HOSTS);
|
|
package("miri-preview", HOSTS);
|
|
package("rustfmt-preview", HOSTS);
|
|
package("rust-analysis", TARGETS);
|
|
package("llvm-tools-preview", TARGETS);
|
|
}
|
|
|
|
fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
|
|
manifest.add_artifact("source-code", |artifact| {
|
|
let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
|
|
artifact.add_tarball(self, "*", &tarball);
|
|
});
|
|
|
|
manifest.add_artifact("installer-msi", |artifact| {
|
|
for target in MSI_INSTALLERS {
|
|
let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
|
|
artifact.add_file(self, target, &msi);
|
|
}
|
|
});
|
|
|
|
manifest.add_artifact("installer-pkg", |artifact| {
|
|
for target in PKG_INSTALLERS {
|
|
let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
|
|
artifact.add_file(self, target, &pkg);
|
|
}
|
|
});
|
|
}
|
|
|
|
fn add_profiles_to(&mut self, manifest: &mut Manifest) {
|
|
let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
|
|
profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
|
|
profile(
|
|
"default",
|
|
&[
|
|
"rustc",
|
|
"cargo",
|
|
"rust-std",
|
|
"rust-mingw",
|
|
"rust-docs",
|
|
"rustfmt-preview",
|
|
"clippy-preview",
|
|
],
|
|
);
|
|
profile(
|
|
"complete",
|
|
&[
|
|
"rustc",
|
|
"cargo",
|
|
"rust-std",
|
|
"rust-mingw",
|
|
"rust-docs",
|
|
"rustfmt-preview",
|
|
"clippy-preview",
|
|
"rls-preview",
|
|
"rust-analyzer-preview",
|
|
"rust-src",
|
|
"llvm-tools-preview",
|
|
"rust-analysis",
|
|
"miri-preview",
|
|
],
|
|
);
|
|
|
|
// The compiler libraries are not stable for end users, and they're also huge, so we only
|
|
// `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
|
|
// for users to install the additional component manually, if needed.
|
|
if self.versions.channel() == "nightly" {
|
|
self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
|
|
self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
|
|
}
|
|
}
|
|
|
|
fn add_renames_to(&self, manifest: &mut Manifest) {
|
|
let mut rename = |from: &str, to: &str| {
|
|
manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
|
|
};
|
|
rename("rls", "rls-preview");
|
|
rename("rustfmt", "rustfmt-preview");
|
|
rename("clippy", "clippy-preview");
|
|
rename("miri", "miri-preview");
|
|
}
|
|
|
|
fn rust_package(&mut self, manifest: &Manifest) -> Package {
|
|
let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
|
|
let mut pkg = Package {
|
|
version: version_info.version.expect("missing Rust version"),
|
|
git_commit_hash: version_info.git_commit,
|
|
target: BTreeMap::new(),
|
|
};
|
|
for host in HOSTS {
|
|
if let Some(target) = self.target_host_combination(host, &manifest) {
|
|
pkg.target.insert(host.to_string(), target);
|
|
} else {
|
|
pkg.target.insert(host.to_string(), Target::unavailable());
|
|
continue;
|
|
}
|
|
}
|
|
pkg
|
|
}
|
|
|
|
fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
|
|
let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
|
|
|
|
let mut target = Target::from_compressed_tar(self, &filename);
|
|
if !target.available {
|
|
return None;
|
|
}
|
|
|
|
let mut components = Vec::new();
|
|
let mut extensions = Vec::new();
|
|
|
|
let host_component = |pkg| Component::from_str(pkg, host);
|
|
|
|
// rustc/rust-std/cargo/docs are all required,
|
|
// and so is rust-mingw if it's available for the target.
|
|
components.extend(vec![
|
|
host_component("rustc"),
|
|
host_component("rust-std"),
|
|
host_component("cargo"),
|
|
host_component("rust-docs"),
|
|
]);
|
|
if host.contains("pc-windows-gnu") {
|
|
components.push(host_component("rust-mingw"));
|
|
}
|
|
|
|
// Tools are always present in the manifest,
|
|
// but might be marked as unavailable if they weren't built.
|
|
extensions.extend(vec![
|
|
host_component("clippy-preview"),
|
|
host_component("miri-preview"),
|
|
host_component("rls-preview"),
|
|
host_component("rust-analyzer-preview"),
|
|
host_component("rustfmt-preview"),
|
|
host_component("llvm-tools-preview"),
|
|
host_component("rust-analysis"),
|
|
]);
|
|
|
|
extensions.extend(
|
|
TARGETS
|
|
.iter()
|
|
.filter(|&&target| target != host)
|
|
.map(|target| Component::from_str("rust-std", target)),
|
|
);
|
|
extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
|
|
extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
|
|
extensions.push(Component::from_str("rust-src", "*"));
|
|
|
|
// If the components/extensions don't actually exist for this
|
|
// particular host/target combination then nix it entirely from our
|
|
// lists.
|
|
let has_component = |c: &Component| {
|
|
if c.target == "*" {
|
|
return true;
|
|
}
|
|
let pkg = match manifest.pkg.get(&c.pkg) {
|
|
Some(p) => p,
|
|
None => return false,
|
|
};
|
|
pkg.target.get(&c.target).is_some()
|
|
};
|
|
extensions.retain(&has_component);
|
|
components.retain(&has_component);
|
|
|
|
target.components = Some(components);
|
|
target.extensions = Some(extensions);
|
|
Some(target)
|
|
}
|
|
|
|
fn profile(
|
|
&mut self,
|
|
profile_name: &str,
|
|
dst: &mut BTreeMap<String, Vec<String>>,
|
|
pkgs: &[&str],
|
|
) {
|
|
dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
|
|
}
|
|
|
|
fn extend_profile(
|
|
&mut self,
|
|
profile_name: &str,
|
|
dst: &mut BTreeMap<String, Vec<String>>,
|
|
pkgs: &[&str],
|
|
) {
|
|
dst.get_mut(profile_name)
|
|
.expect("existing profile")
|
|
.extend(pkgs.iter().map(|s| (*s).to_owned()));
|
|
}
|
|
|
|
fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
|
|
let version_info = self
|
|
.versions
|
|
.version(&PkgType::from_component(pkgname))
|
|
.expect("failed to load package version");
|
|
let mut is_present = version_info.present;
|
|
|
|
// Never ship nightly-only components for other trains.
|
|
if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
|
|
is_present = false; // Pretend the component is entirely missing.
|
|
}
|
|
|
|
let targets = targets
|
|
.iter()
|
|
.map(|name| {
|
|
let target = if is_present {
|
|
let filename = self
|
|
.versions
|
|
.tarball_name(&PkgType::from_component(pkgname), name)
|
|
.unwrap();
|
|
|
|
Target::from_compressed_tar(self, &filename)
|
|
} else {
|
|
// If the component is not present for this build add it anyway but mark it as
|
|
// unavailable -- this way rustup won't allow upgrades without --force
|
|
Target::unavailable()
|
|
};
|
|
(name.to_string(), target)
|
|
})
|
|
.collect();
|
|
|
|
dst.insert(
|
|
pkgname.to_string(),
|
|
Package {
|
|
version: version_info.version.unwrap_or_default(),
|
|
git_commit_hash: version_info.git_commit,
|
|
target: targets,
|
|
},
|
|
);
|
|
}
|
|
|
|
fn url(&self, path: &Path) -> String {
|
|
let file_name = path.file_name().unwrap().to_str().unwrap();
|
|
format!("{}/{}/{}", self.s3_address, self.date, file_name)
|
|
}
|
|
|
|
fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) {
|
|
self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
|
|
self.write(&manifest.date, channel_name, "-date.txt");
|
|
self.write(
|
|
manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
|
|
channel_name,
|
|
"-git-commit-hash.txt",
|
|
);
|
|
}
|
|
|
|
fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) {
|
|
let name = format!("channel-rust-{}{}", channel_name, suffix);
|
|
self.shipped_files.insert(name.clone());
|
|
|
|
let dst = self.output.join(name);
|
|
t!(fs::write(&dst, contents));
|
|
}
|
|
|
|
fn write_shipped_files(&self, path: &Path) {
|
|
let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::<Vec<_>>();
|
|
files.sort();
|
|
let content = format!("{}\n", files.join("\n"));
|
|
|
|
t!(std::fs::write(path, content.as_bytes()));
|
|
}
|
|
}
|