OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Auto merge of #77145 - pietroalbini:refactor-build-manifest-versions, r=Mark-Simulacrum 

Refactor versions detection in build-manifest

This PR refactors how `build-manifest` handles versions, making the following changes:

* `build-manifest` now detects the "package releases" on its own, without relying on rustbuild providing them through CLI arguments. This drastically simplifies calling the tool outside of `x.py`, and will allow to ship the prebuilt tool in a tarball in the future, with the goal of stopping to invoke `x.py` during `promote-release`.
* The `tar` command is not used to extract the version and the git hash from tarballs anymore. The `flate2` and `tar` crates are used instead. This makes detecting those pieces of data way faster, as the archive is decompressed just once and we stop parsing the archive once all the information is retrieved.
* The code to extract the version and the git hash now stores all the collected data dynamically, without requiring to add new fields to the `Builder` struct every time.

I tested the changes locally and it should behave the same as before.

r? `@Mark-Simulacrum`
This commit is contained in:
bors 2020-09-29 16:41:53 +00:00
parent 4d52dc4790 0749ad02d0
commit 9b77a6a200
6 changed files with 293 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock
View File

@ -231,8 +231,11 @@ dependencies = [
name = "build-manifest"
version = "0.1.0"
dependencies = [
"anyhow",
"flate2",
"serde",
"serde_json",
"tar",
"toml",
]

10
src/bootstrap/dist.rs
View File

@ -2368,15 +2368,9 @@ impl Step for HashSign {
cmd.arg(sign);
cmd.arg(distdir(builder));
cmd.arg(today.trim());
cmd.arg(builder.rust_package_vers());
cmd.arg(addr);
cmd.arg(builder.package_vers(&builder.release_num("cargo")));
cmd.arg(builder.package_vers(&builder.release_num("rls")));
cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer")));
cmd.arg(builder.package_vers(&builder.release_num("clippy")));
cmd.arg(builder.package_vers(&builder.release_num("miri")));
cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
cmd.arg(builder.llvm_tools_package_vers());
cmd.arg(&builder.config.channel);
cmd.arg(&builder.src);
builder.create_dir(&distdir(builder));

3
src/tools/build-manifest/Cargo.toml
View File

@ -8,3 +8,6 @@ edition = "2018"
toml = "0.5"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
anyhow = "1.0.32"
flate2 = "1.0.16"
tar = "0.4.29"

9
src/tools/build-manifest/README.md
View File

@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
```
$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
path/to/dist path/to/output 1970-01-01 \
nightly nightly nightly nightly nightly nightly nightly nightly \
http://example.com
path/to/dist path/to/output 1970-01-01 http://example.com \
CHANNEL path/to/rust/repo
```
In the future, if the tool complains about missing arguments just add more
`nightly`s in the middle.
Remember to replace `CHANNEL` with the channel you produced dist artifacts of
and `path/to/rust/repo` with the path to your checkout of the Rust repository.

233
src/tools/build-manifest/src/main.rs
View File

@ -4,8 +4,10 @@
//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
//! by rustbuild (in `src/bootstrap/dist.rs`).
use serde::Serialize;
mod versions;
use crate::versions::{PkgType, Versions};
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::env;
@ -226,14 +228,7 @@ macro_rules! t {
}
struct Builder {
rust_release: String,
cargo_release: String,
rls_release: String,
rust_analyzer_release: String,
clippy_release: String,
rustfmt_release: String,
llvm_tools_release: String,
miri_release: String,
versions: Versions,
input: PathBuf,
output: PathBuf,
@ -242,24 +237,6 @@ struct Builder {
s3_address: String,
date: String,
rust_version: Option<String>,
cargo_version: Option<String>,
rls_version: Option<String>,
rust_analyzer_version: Option<String>,
clippy_version: Option<String>,
rustfmt_version: Option<String>,
llvm_tools_version: Option<String>,
miri_version: Option<String>,
rust_git_commit_hash: Option<String>,
cargo_git_commit_hash: Option<String>,
rls_git_commit_hash: Option<String>,
rust_analyzer_git_commit_hash: Option<String>,
clippy_git_commit_hash: Option<String>,
rustfmt_git_commit_hash: Option<String>,
llvm_tools_git_commit_hash: Option<String>,
miri_git_commit_hash: Option<String>,
should_sign: bool,
}
@ -280,15 +257,9 @@ fn main() {
let input = PathBuf::from(args.next().unwrap());
let output = PathBuf::from(args.next().unwrap());
let date = args.next().unwrap();
let rust_release = args.next().unwrap();
let s3_address = args.next().unwrap();
let cargo_release = args.next().unwrap();
let rls_release = args.next().unwrap();
let rust_analyzer_release = args.next().unwrap();
let clippy_release = args.next().unwrap();
let miri_release = args.next().unwrap();
let rustfmt_release = args.next().unwrap();
let llvm_tools_release = args.next().unwrap();
let channel = args.next().unwrap();
let monorepo_path = args.next().unwrap();
// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
@ -298,14 +269,7 @@ fn main() {
}
Builder {
rust_release,
cargo_release,
rls_release,
rust_analyzer_release,
clippy_release,
rustfmt_release,
llvm_tools_release,
miri_release,
versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
input,
output,
@ -314,87 +278,21 @@ fn main() {
s3_address,
date,
rust_version: None,
cargo_version: None,
rls_version: None,
rust_analyzer_version: None,
clippy_version: None,
rustfmt_version: None,
llvm_tools_version: None,
miri_version: None,
rust_git_commit_hash: None,
cargo_git_commit_hash: None,
rls_git_commit_hash: None,
rust_analyzer_git_commit_hash: None,
clippy_git_commit_hash: None,
rustfmt_git_commit_hash: None,
llvm_tools_git_commit_hash: None,
miri_git_commit_hash: None,
should_sign,
}
.build();
}
enum PkgType {
RustSrc,
Cargo,
Rls,
RustAnalyzer,
Clippy,
Rustfmt,
LlvmTools,
Miri,
Other,
}
impl PkgType {
fn from_component(component: &str) -> Self {
use PkgType::*;
match component {
"rust-src" => RustSrc,
"cargo" => Cargo,
"rls" | "rls-preview" => Rls,
"rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer,
"clippy" | "clippy-preview" => Clippy,
"rustfmt" | "rustfmt-preview" => Rustfmt,
"llvm-tools" | "llvm-tools-preview" => LlvmTools,
"miri" | "miri-preview" => Miri,
_ => Other,
}
}
}
impl Builder {
fn build(&mut self) {
self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu");
self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");
self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
self.rust_analyzer_git_commit_hash =
self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu");
self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
self.llvm_tools_git_commit_hash =
self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu");
self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
self.check_toolstate();
self.digest_and_sign();
let manifest = self.build_manifest();
self.write_channel_files(&self.rust_release, &manifest);
if self.rust_release != "beta" && self.rust_release != "nightly" {
self.write_channel_files("stable", &manifest);
let rust_version = self.versions.package_version(&PkgType::Rust).unwrap();
self.write_channel_files(self.versions.channel(), &manifest);
if self.versions.channel() != rust_version {
self.write_channel_files(&rust_version, &manifest);
}
}
@ -415,8 +313,7 @@ impl Builder {
// Mark some tools as missing based on toolstate.
if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
println!("Miri tests are not passing, removing component");
self.miri_version = None;
self.miri_git_commit_hash = None;
self.versions.disable_version(&PkgType::Miri);
}
}
@ -501,7 +398,7 @@ impl Builder {
// The compiler libraries are not stable for end users, and they're also huge, so we only
// `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
// for users to install the additional component manually, if needed.
if self.rust_release == "nightly" {
if self.versions.channel() == "nightly" {
self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
}
@ -518,13 +415,10 @@ impl Builder {
}
fn rust_package(&mut self, manifest: &Manifest) -> Package {
let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
let mut pkg = Package {
version: self
.cached_version("rust")
.as_ref()
.expect("Couldn't find Rust version")
.clone(),
git_commit_hash: self.cached_git_commit_hash("rust").clone(),
version: version_info.version.expect("missing Rust version"),
git_commit_hash: version_info.git_commit,
target: BTreeMap::new(),
};
for host in HOSTS {
@ -539,7 +433,7 @@ impl Builder {
}
fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
let filename = self.filename("rust", host);
let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
let digest = self.digests.remove(&filename)?;
let xz_filename = filename.replace(".tar.gz", ".tar.xz");
let xz_digest = self.digests.remove(&xz_filename);
@ -630,15 +524,14 @@ impl Builder {
}
fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
let (version, mut is_present) = self
.cached_version(pkgname)
.as_ref()
.cloned()
.map(|version| (version, true))
.unwrap_or_default(); // `is_present` defaults to `false` here.
let version_info = self
.versions
.version(&PkgType::from_component(pkgname))
.expect("failed to load package version");
let mut is_present = version_info.present;
// Never ship nightly-only components for other trains.
if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
is_present = false; // Pretend the component is entirely missing.
}
@ -647,7 +540,10 @@ impl Builder {
.map(|name| {
if is_present {
// The component generally exists, but it might still be missing for this target.
let filename = self.filename(pkgname, name);
let filename = self
.versions
.tarball_name(&PkgType::from_component(pkgname), name)
.unwrap();
let digest = match self.digests.remove(&filename) {
Some(digest) => digest,
// This component does not exist for this target -- skip it.
@ -679,8 +575,8 @@ impl Builder {
dst.insert(
pkgname.to_string(),
Package {
version,
git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
version: version_info.version.unwrap_or_default(),
git_commit_hash: version_info.git_commit,
target: targets,
},
);
@ -690,77 +586,6 @@ impl Builder {
format!("{}/{}/{}", self.s3_address, self.date, filename)
}
fn filename(&self, component: &str, target: &str) -> String {
use PkgType::*;
match PkgType::from_component(component) {
RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
RustAnalyzer => {
format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target)
}
Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
}
}
fn cached_version(&self, component: &str) -> &Option<String> {
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_version,
Rls => &self.rls_version,
RustAnalyzer => &self.rust_analyzer_version,
Clippy => &self.clippy_version,
Rustfmt => &self.rustfmt_version,
LlvmTools => &self.llvm_tools_version,
Miri => &self.miri_version,
_ => &self.rust_version,
}
}
fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_git_commit_hash,
Rls => &self.rls_git_commit_hash,
RustAnalyzer => &self.rust_analyzer_git_commit_hash,
Clippy => &self.clippy_git_commit_hash,
Rustfmt => &self.rustfmt_git_commit_hash,
LlvmTools => &self.llvm_tools_git_commit_hash,
Miri => &self.miri_git_commit_hash,
_ => &self.rust_git_commit_hash,
}
}
fn version(&self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/version", filename))
}
fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
}
fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
where
F: FnOnce(String) -> String,
{
let mut cmd = Command::new("tar");
let filename = self.filename(component, target);
cmd.arg("xf")
.arg(self.input.join(&filename))
.arg(dir(filename.replace(".tar.gz", "")))
.arg("-O");
let output = t!(cmd.output());
if output.status.success() {
Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
} else {
None
}
}
fn hash(&self, path: &Path) -> String {
let sha = t!(Command::new("shasum")
.arg("-a")

252
src/tools/build-manifest/src/versions.rs Normal file
View File

@ -0,0 +1,252 @@
use anyhow::{Context, Error};
use flate2::read::GzDecoder;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use tar::Archive;
const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
#[derive(Debug, Hash, Eq, PartialEq, Clone)]
pub(crate) enum PkgType {
Rust,
RustSrc,
Cargo,
Rls,
RustAnalyzer,
Clippy,
Rustfmt,
LlvmTools,
Miri,
Other(String),
}
impl PkgType {
pub(crate) fn from_component(component: &str) -> Self {
match component {
"rust" => PkgType::Rust,
"rust-src" => PkgType::RustSrc,
"cargo" => PkgType::Cargo,
"rls" | "rls-preview" => PkgType::Rls,
"rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
"clippy" | "clippy-preview" => PkgType::Clippy,
"rustfmt" | "rustfmt-preview" => PkgType::Rustfmt,
"llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools,
"miri" | "miri-preview" => PkgType::Miri,
other => PkgType::Other(other.into()),
}
}
/// The directory containing the `Cargo.toml` of this component inside the monorepo, to
/// retrieve the source code version. If `None` is returned Rust's version will be used.
fn rust_monorepo_path(&self) -> Option<&'static str> {
match self {
PkgType::Cargo => Some("src/tools/cargo"),
PkgType::Rls => Some("src/tools/rls"),
PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"),
PkgType::Clippy => Some("src/tools/clippy"),
PkgType::Rustfmt => Some("src/tools/rustfmt"),
PkgType::Miri => Some("src/tools/miri"),
PkgType::Rust => None,
PkgType::RustSrc => None,
PkgType::LlvmTools => None,
PkgType::Other(_) => None,
}
}
/// First part of the tarball name.
fn tarball_component_name(&self) -> &str {
match self {
PkgType::Rust => "rust",
PkgType::RustSrc => "rust-src",
PkgType::Cargo => "cargo",
PkgType::Rls => "rls",
PkgType::RustAnalyzer => "rust-analyzer",
PkgType::Clippy => "clippy",
PkgType::Rustfmt => "rustfmt",
PkgType::LlvmTools => "llvm-tools",
PkgType::Miri => "miri",
PkgType::Other(component) => component,
}
}
/// Whether this package has the same version as Rust itself, or has its own `version` and
/// `git-commit-hash` files inside the tarball.
fn should_use_rust_version(&self) -> bool {
match self {
PkgType::Cargo => false,
PkgType::Rls => false,
PkgType::RustAnalyzer => false,
PkgType::Clippy => false,
PkgType::Rustfmt => false,
PkgType::LlvmTools => false,
PkgType::Miri => false,
PkgType::Rust => true,
PkgType::RustSrc => true,
PkgType::Other(_) => true,
}
}
/// Whether this package is target-independent or not.
fn target_independent(&self) -> bool {
*self == PkgType::RustSrc
}
}
#[derive(Debug, Default, Clone)]
pub(crate) struct VersionInfo {
pub(crate) version: Option<String>,
pub(crate) git_commit: Option<String>,
pub(crate) present: bool,
}
pub(crate) struct Versions {
channel: String,
rustc_version: String,
monorepo_root: PathBuf,
dist_path: PathBuf,
package_versions: HashMap<PkgType, String>,
versions: HashMap<PkgType, VersionInfo>,
}
impl Versions {
pub(crate) fn new(
channel: &str,
dist_path: &Path,
monorepo_root: &Path,
) -> Result<Self, Error> {
Ok(Self {
channel: channel.into(),
rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
.context("failed to read the rustc version from src/version")?
.trim()
.to_string(),
monorepo_root: monorepo_root.into(),
dist_path: dist_path.into(),
package_versions: HashMap::new(),
versions: HashMap::new(),
})
}
pub(crate) fn channel(&self) -> &str {
&self.channel
}
pub(crate) fn version(&mut self, mut package: &PkgType) -> Result<VersionInfo, Error> {
if package.should_use_rust_version() {
package = &PkgType::Rust;
}
match self.versions.get(package) {
Some(version) => Ok(version.clone()),
None => {
let version_info = self.load_version_from_tarball(package)?;
self.versions.insert(package.clone(), version_info.clone());
Ok(version_info)
}
}
}
fn load_version_from_tarball(&mut self, package: &PkgType) -> Result<VersionInfo, Error> {
let tarball_name = self.tarball_name(package, DEFAULT_TARGET)?;
let tarball = self.dist_path.join(tarball_name);
let file = match File::open(&tarball) {
Ok(file) => file,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
// Missing tarballs do not return an error, but return empty data.
return Ok(VersionInfo::default());
}
Err(err) => return Err(err.into()),
};
let mut tar = Archive::new(GzDecoder::new(file));
let mut version = None;
let mut git_commit = None;
for entry in tar.entries()? {
let mut entry = entry?;
let dest;
match entry.path()?.components().nth(1).and_then(|c| c.as_os_str().to_str()) {
Some("version") => dest = &mut version,
Some("git-commit-hash") => dest = &mut git_commit,
_ => continue,
}
let mut buf = String::new();
entry.read_to_string(&mut buf)?;
*dest = Some(buf);
// Short circuit to avoid reading the whole tar file if not necessary.
if version.is_some() && git_commit.is_some() {
break;
}
}
Ok(VersionInfo { version, git_commit, present: true })
}
pub(crate) fn disable_version(&mut self, package: &PkgType) {
match self.versions.get_mut(package) {
Some(version) => {
*version = VersionInfo::default();
}
None => {
self.versions.insert(package.clone(), VersionInfo::default());
}
}
}
pub(crate) fn tarball_name(
&mut self,
package: &PkgType,
target: &str,
) -> Result<String, Error> {
let component_name = package.tarball_component_name();
let version = self.package_version(package).with_context(|| {
format!("failed to get the package version for component {:?}", package,)
})?;
if package.target_independent() {
Ok(format!("{}-{}.tar.gz", component_name, version))
} else {
Ok(format!("{}-{}-{}.tar.gz", component_name, version, target))
}
}
pub(crate) fn package_version(&mut self, package: &PkgType) -> Result<String, Error> {
match self.package_versions.get(package) {
Some(release) => Ok(release.clone()),
None => {
let version = match package.rust_monorepo_path() {
Some(path) => {
let path = self.monorepo_root.join(path).join("Cargo.toml");
let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?;
cargo_toml.package.version
}
None => self.rustc_version.clone(),
};
let release = match self.channel.as_str() {
"stable" => version,
"beta" => "beta".into(),
"nightly" => "nightly".into(),
_ => format!("{}-dev", version),
};
self.package_versions.insert(package.clone(), release.clone());
Ok(release)
}
}
}
}
#[derive(serde::Deserialize)]
struct CargoToml {
package: CargoTomlPackage,
}
#[derive(serde::Deserialize)]
struct CargoTomlPackage {
version: String,
}