From 9e8785f01757a50e321e85adeb659a9aff6fc00e Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 24 Jan 2017 14:37:04 -0800 Subject: [PATCH] rustbuild: Add manifest generation in-tree This commit adds a new tool, `build-manifest`, which is used to generate a distribution manifest of all produced artifacts. This tool is intended to replace the `build-rust-manifest.py` script that's currently located on the buildmaster. The intention is that we'll have a builder which periodically: * Downloads all artifacts for a commit * Runs `./x.py dist hash-and-sign`. This will generate `sha256` and `asc` files as well as TOML manifests. * Upload all generated hashes and manifests to the directory the artifacts came from. * Upload *all* artifacts (tarballs and hashes and manifests) to an archived location. * If necessary, upload all artifacts to the main location. This script is intended to just be the second step here where orchestrating uploads and such will all happen externally from the build system itself. --- src/Cargo.lock | 8 + src/Cargo.toml | 1 + src/bootstrap/config.rs | 19 ++ src/bootstrap/config.toml.example | 30 ++ src/bootstrap/dist.rs | 33 ++- src/bootstrap/step.rs | 10 + src/tools/build-manifest/Cargo.toml | 8 + src/tools/build-manifest/src/main.rs | 404 +++++++++++++++++++++++++++ 8 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 src/tools/build-manifest/Cargo.toml create mode 100644 src/tools/build-manifest/src/main.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 7db243c5eb9..93bbf0f227b 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -50,6 +50,14 @@ dependencies = [ "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "build-manifest" +version = "0.1.0" +dependencies = [ + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "build_helper" version = "0.1.0" diff --git a/src/Cargo.toml b/src/Cargo.toml index 8fb5c70c41b..0db26ea5ae0 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -10,6 +10,7 @@ members = [ "tools/linkchecker", "tools/rustbook", "tools/tidy", + "tools/build-manifest", ] # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7d1abcfa6f6..e035f8157ff 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -78,6 +78,11 @@ pub struct Config { pub cargo: Option, pub local_rebuild: bool, + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_gpg_password_file: Option, + // libstd features pub debug_jemalloc: bool, pub use_jemalloc: bool, @@ -123,6 +128,7 @@ struct TomlConfig { llvm: Option, rust: Option, target: Option>, + dist: Option, } /// TOML representation of various global build decisions. @@ -166,6 +172,13 @@ struct Llvm { targets: Option, } +#[derive(RustcDecodable, Default, Clone)] +struct Dist { + sign_folder: Option, + gpg_password_file: Option, + upload_addr: Option, +} + #[derive(RustcDecodable)] enum StringOrBool { String(String), @@ -352,6 +365,12 @@ impl Config { } } + if let Some(ref t) = toml.dist { + config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); + config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); + config.dist_upload_addr = t.upload_addr.clone(); + } + return config } diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 4b859482562..a53419ad7fd 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -242,3 +242,33 @@ # that this option only makes sense for MUSL targets that produce statically # linked binaries #musl-root = "..." + +# ============================================================================= +# Distribution options +# +# These options are related to distribution, mostly for the Rust project itself. +# You probably won't need to concern yourself with any of these options +# ============================================================================= +[dist] + +# This is the folder of artifacts that the build system will sign. All files in +# this directory will be signed with the default gpg key using the system `gpg` +# binary. The `asc` and `sha256` files will all be output into the standard dist +# output folder (currently `build/dist`) +# +# This folder should be populated ahead of time before the build system is +# invoked. +#sign-folder = "path/to/folder/to/sign" + +# This is a file which contains the password of the default gpg key. This will +# be passed to `gpg` down the road when signing all files in `sign-folder` +# above. This should be stored in plaintext. +#gpg-password-file = "path/to/gpg/password" + +# The remote address that all artifacts will eventually be uploaded to. The +# build system generates manifests which will point to these urls, and for the +# manifests to be correct they'll have to have the right URLs encoded. +# +# Note that this address should not contain a trailing slash as file names will +# be appended to it. +#upload-addr = "https://example.com/folder" diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e5f05059523..71a5f313bbd 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -22,7 +22,7 @@ use std::env; use std::fs::{self, File}; use std::io::{Read, Write}; use std::path::{PathBuf, Path}; -use std::process::Command; +use std::process::{Command, Stdio}; use build_helper::output; @@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) { cmd.env("CFG_PLATFORM", "x86"); } } + +pub fn hash_and_sign(build: &Build) { + let compiler = Compiler::new(0, &build.config.build); + let mut cmd = build.tool_cmd(&compiler, "build-manifest"); + let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n") + }); + let mut pass = String::new(); + t!(t!(File::open(&file)).read_to_string(&mut pass)); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(build)); + cmd.arg(today.trim()); + cmd.arg(package_vers(build)); + cmd.arg(addr); + + t!(fs::create_dir_all(distdir(build))); + + let mut child = t!(cmd.stdin(Stdio::piped()).spawn()); + t!(child.stdin.take().unwrap().write_all(pass.as_bytes())); + let status = t!(child.wait()); + assert!(status.success()); +} diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 697b14c6050..3932a7cf8c5 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build("tool-compiletest", "src/tools/compiletest") .dep(|s| s.name("libtest")) .run(move |s| compile::tool(build, s.stage, s.target, "compiletest")); + rules.build("tool-build-manifest", "src/tools/build-manifest") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest")); // ======================================================================== // Documentation targets @@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|d| d.name("dist-cargo")) .run(move |s| dist::extended(build, s.stage, s.target)); + rules.dist("dist-sign", "hash-and-sign") + .host(true) + .only_build(true) + .only_host_build(true) + .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0)) + .run(move |_| dist::hash_and_sign(build)); + rules.verify(); return rules; } diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml new file mode 100644 index 00000000000..4b876753b1f --- /dev/null +++ b/src/tools/build-manifest/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "build-manifest" +version = "0.1.0" +authors = ["Alex Crichton "] + +[dependencies] +toml = "0.1" +rustc-serialize = "0.3" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs new file mode 100644 index 00000000000..8c15a6630a3 --- /dev/null +++ b/src/tools/build-manifest/src/main.rs @@ -0,0 +1,404 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate toml; +extern crate rustc_serialize; + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::path::{PathBuf, Path}; +use std::process::{Command, Stdio}; + +static HOSTS: &'static [&'static str] = &[ + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "i686-apple-darwin", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-netbsd", +]; + +static TARGETS: &'static [&'static str] = &[ + "aarch64-apple-ios", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "arm-linux-androideabi", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "arm-unknown-linux-musleabi", + "arm-unknown-linux-musleabihf", + "armv7-apple-ios", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabihf", + "armv7-unknown-linux-musleabihf", + "armv7s-apple-ios", + "asmjs-unknown-emscripten", + "i386-apple-ios", + "i586-pc-windows-msvc", + "i586-unknown-linux-gnu", + "i686-apple-darwin", + "i686-linux-android", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "i686-unknown-linux-musl", + "mips-unknown-linux-gnu", + "mips-unknown-linux-musl", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "mipsel-unknown-linux-musl", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "wasm32-unknown-emscripten", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-rumprun-netbsd", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", +]; + +static MINGW: &'static [&'static str] = &[ + "i686-pc-windows-gnu", + "x86_64-pc-windows-gnu", +]; + +#[derive(RustcEncodable)] +struct Manifest { + manifest_version: String, + date: String, + pkg: HashMap, +} + +#[derive(RustcEncodable)] +struct Package { + version: String, + target: HashMap, +} + +#[derive(RustcEncodable)] +struct Target { + available: bool, + url: Option, + hash: Option, + components: Option>, + extensions: Option>, +} + +#[derive(RustcEncodable)] +struct Component { + pkg: String, + target: String, +} + +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +struct Builder { + channel: String, + input: PathBuf, + output: PathBuf, + gpg_passphrase: String, + digests: HashMap, + s3_address: String, + date: String, + rust_version: String, + cargo_version: String, +} + +fn main() { + let mut args = env::args().skip(1); + let input = PathBuf::from(args.next().unwrap()); + let output = PathBuf::from(args.next().unwrap()); + let date = args.next().unwrap(); + let channel = args.next().unwrap(); + let s3_address = args.next().unwrap(); + let mut passphrase = String::new(); + t!(io::stdin().read_to_string(&mut passphrase)); + + Builder { + channel: channel, + input: input, + output: output, + gpg_passphrase: passphrase, + digests: HashMap::new(), + s3_address: s3_address, + date: date, + rust_version: String::new(), + cargo_version: String::new(), + }.build(); +} + +impl Builder { + fn build(&mut self) { + self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); + self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu"); + + self.digest_and_sign(); + let manifest = self.build_manifest(); + let manifest = toml::encode(&manifest).to_string(); + + let filename = format!("channel-rust-{}.toml", self.channel); + self.write_manifest(&manifest, &filename); + + if self.channel != "beta" && self.channel != "nightly" { + self.write_manifest(&manifest, "channel-rust-stable.toml"); + } + } + + fn digest_and_sign(&mut self) { + for file in t!(self.input.read_dir()).map(|e| t!(e).path()) { + let filename = file.file_name().unwrap().to_str().unwrap(); + let digest = self.hash(&file); + self.sign(&file); + assert!(self.digests.insert(filename.to_string(), digest).is_none()); + } + } + + fn build_manifest(&mut self) -> Manifest { + let mut manifest = Manifest { + manifest_version: "2".to_string(), + date: self.date.to_string(), + pkg: HashMap::new(), + }; + + self.package("rustc", &mut manifest.pkg, HOSTS); + self.package("cargo", &mut manifest.pkg, HOSTS); + self.package("rust-mingw", &mut manifest.pkg, MINGW); + self.package("rust-std", &mut manifest.pkg, TARGETS); + self.package("rust-docs", &mut manifest.pkg, TARGETS); + self.package("rust-src", &mut manifest.pkg, &["*"]); + + let mut pkg = Package { + version: self.cached_version("rust").to_string(), + target: HashMap::new(), + }; + for host in HOSTS { + let filename = self.filename("rust", host); + let digest = match self.digests.remove(&filename) { + Some(digest) => digest, + None => { + pkg.target.insert(host.to_string(), Target { + available: false, + url: None, + hash: None, + components: None, + extensions: None, + }); + continue + } + }; + let mut components = Vec::new(); + let mut extensions = Vec::new(); + + // rustc/rust-std/cargo are all required, and so is rust-mingw if it's + // available for the target. + components.extend(vec![ + Component { pkg: "rustc".to_string(), target: host.to_string() }, + Component { pkg: "rust-std".to_string(), target: host.to_string() }, + Component { pkg: "cargo".to_string(), target: host.to_string() }, + ]); + if host.contains("pc-windows-gnu") { + components.push(Component { + pkg: "rust-mingw".to_string(), + target: host.to_string(), + }); + } + + // Docs, other standard libraries, and the source package are all + // optional. + extensions.push(Component { + pkg: "rust-docs".to_string(), + target: host.to_string(), + }); + for target in TARGETS { + if target != host { + extensions.push(Component { + pkg: "rust-std".to_string(), + target: target.to_string(), + }); + } + } + extensions.push(Component { + pkg: "rust-src".to_string(), + target: "*".to_string(), + }); + + pkg.target.insert(host.to_string(), Target { + available: true, + url: Some(self.url("rust", host)), + hash: Some(to_hex(digest.as_ref())), + components: Some(components), + extensions: Some(extensions), + }); + } + manifest.pkg.insert("rust".to_string(), pkg); + + return manifest + } + + fn package(&mut self, + pkgname: &str, + dst: &mut HashMap, + targets: &[&str]) { + let targets = targets.iter().map(|name| { + let filename = self.filename(pkgname, name); + let digest = match self.digests.remove(&filename) { + Some(digest) => digest, + None => { + return (name.to_string(), Target { + available: false, + url: None, + hash: None, + components: None, + extensions: None, + }) + } + }; + + (name.to_string(), Target { + available: true, + url: Some(self.url(pkgname, name)), + hash: Some(digest), + components: None, + extensions: None, + }) + }).collect(); + + dst.insert(pkgname.to_string(), Package { + version: self.cached_version(pkgname).to_string(), + target: targets, + }); + } + + fn url(&self, component: &str, target: &str) -> String { + format!("{}/{}/{}", + self.s3_address, + self.date, + self.filename(component, target)) + } + + fn filename(&self, component: &str, target: &str) -> String { + if component == "rust-src" { + format!("rust-src-{}.tar.gz", self.channel) + } else { + format!("{}-{}-{}.tar.gz", component, self.channel, target) + } + } + + fn cached_version(&self, component: &str) -> &str { + if component == "cargo" { + &self.cargo_version + } else { + &self.rust_version + } + } + + fn version(&self, component: &str, target: &str) -> String { + let mut cmd = Command::new("tar"); + let filename = self.filename(component, target); + cmd.arg("xf") + .arg(self.input.join(&filename)) + .arg(format!("{}/version", filename.replace(".tar.gz", ""))) + .arg("-O"); + let version = t!(cmd.output()); + if !version.status.success() { + panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}", + cmd, + String::from_utf8_lossy(&version.stdout), + String::from_utf8_lossy(&version.stderr)); + } + String::from_utf8_lossy(&version.stdout).trim().to_string() + } + + fn hash(&self, path: &Path) -> String { + let sha = t!(Command::new("shasum") + .arg("-a").arg("256") + .arg(path) + .output()); + assert!(sha.status.success()); + + let filename = path.file_name().unwrap().to_str().unwrap(); + let sha256 = self.output.join(format!("{}.sha256", filename)); + t!(t!(File::create(&sha256)).write_all(&sha.stdout)); + + let stdout = String::from_utf8_lossy(&sha.stdout); + stdout.split_whitespace().next().unwrap().to_string() + } + + fn sign(&self, path: &Path) { + let filename = path.file_name().unwrap().to_str().unwrap(); + let asc = self.output.join(format!("{}.asc", filename)); + println!("signing: {:?}", path); + let mut cmd = Command::new("gpg"); + cmd.arg("--no-tty") + .arg("--yes") + .arg("--passphrase-fd").arg("0") + .arg("--armor") + .arg("--output").arg(&asc) + .arg("--detach-sign").arg(path) + .stdin(Stdio::piped()); + let mut child = t!(cmd.spawn()); + t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes())); + assert!(t!(child.wait()).success()); + } + + fn write_manifest(&self, manifest: &str, name: &str) { + let dst = self.output.join(name); + t!(t!(File::create(&dst)).write_all(manifest.as_bytes())); + self.hash(&dst); + self.sign(&dst); + } +} + +fn to_hex(digest: &[u8]) -> String { + let mut ret = String::new(); + for byte in digest { + ret.push(hex((byte & 0xf0) >> 4)); + ret.push(hex(byte & 0xf)); + } + return ret; + + fn hex(b: u8) -> char { + match b { + 0...9 => (b'0' + b) as char, + _ => (b'a' + b - 10) as char, + } + } +}