Auto merge of #53803 - pietroalbini:fix-manifest-2, r=alexcrichton

Fix manifests for broken tools: take 2

This is a follow up of #53715, to avoid stripping unavailable components from the extensions list. This time I also figured out how to test the changes, so the produced manifest is correct.

Along with the fix I added a README with instructions on how to test the tool, and a new `BUILD_MANIFEST_DISABLE_SIGNING` env var to avoid dealing with gpg while testing the tool. I chose an env var instead of a flag because it's more difficult to have it slip in by accident on CI, and there is also another protection that panics if that env var is set on CI, just to be sure we don't release unsigned artifacts.

r? @alexcrichton
cc https://github.com/rust-lang-nursery/rustup.rs/issues/1486

src/tools/build-manifest/README.md

@@ -0,0 +1,30 @@
+# build-manifest
+
+This tool generates the manifests uploaded to static.rust-lang.org and used by
+rustup. The tool is invoked by the bootstrap tool.
+
+## Testing changes locally
+
+In order to test the changes locally you need to have a valid dist directory
+available locally. If you don't want to build all the compiler, you can easily
+create one from the nightly artifacts with:
+
+```
+#!/bin/bash
+for cmpn in rust rustc rust-std rust-docs cargo; do
+    wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz
+done
+```
+
+Then, you can generate the manifest and all the packages from `path/to/dist` to
+`path/to/output` with:
+
+```
+$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
+    path/to/dist path/to/output 1970-01-01 \
+    nightly nightly nightly nightly nightly nightly nightly \
+    http://example.com
+```
+
+In the future, if the tool complains about missing arguments just add more
+`nightly`s in the middle.

src/tools/build-manifest/src/main.rs

@@ -121,6 +121,17 @@ static TARGETS: &'static [&'static str] = &[
     "x86_64-unknown-redox",
 ];
 
+static DOCS_TARGETS: &'static [&'static str] = &[
+    "i686-apple-darwin",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "i686-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+    "x86_64-unknown-linux-gnu",
+];
+
 static MINGW: &'static [&'static str] = &[
     "i686-pc-windows-gnu",
     "x86_64-pc-windows-gnu",
@@ -216,9 +227,23 @@ struct Builder {
     rustfmt_git_commit_hash: Option<String>,
     llvm_tools_git_commit_hash: Option<String>,
     lldb_git_commit_hash: Option<String>,
+
+    should_sign: bool,
 }
 
 fn main() {
+    // Avoid signing packages while manually testing
+    // Do NOT set this envvar in CI
+    let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
+
+    // Safety check to ensure signing is always enabled on CI
+    // The CI environment variable is set by both Travis and AppVeyor
+    if !should_sign && env::var("CI").is_ok() {
+        println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
+        println!("If you're not running this on CI, unset the 'CI' env var.");
+        panic!();
+    }
+
     let mut args = env::args().skip(1);
     let input = PathBuf::from(args.next().unwrap());
     let output = PathBuf::from(args.next().unwrap());
@@ -231,8 +256,12 @@ fn main() {
     let llvm_tools_release = args.next().unwrap();
     let lldb_release = args.next().unwrap();
     let s3_address = args.next().unwrap();
+
+    // Do not ask for a passphrase while manually testing
     let mut passphrase = String::new();
-    t!(io::stdin().read_to_string(&mut passphrase));
+    if should_sign {
+        t!(io::stdin().read_to_string(&mut passphrase));
+    }
 
     Builder {
         rust_release,
@@ -265,6 +294,8 @@ fn main() {
         rustfmt_git_commit_hash: None,
         llvm_tools_git_commit_hash: None,
         lldb_git_commit_hash: None,
+
+        should_sign,
     }.build();
 }
 
@@ -318,7 +349,7 @@ impl Builder {
         self.package("cargo", &mut manifest.pkg, HOSTS);
         self.package("rust-mingw", &mut manifest.pkg, MINGW);
         self.package("rust-std", &mut manifest.pkg, TARGETS);
-        self.package("rust-docs", &mut manifest.pkg, TARGETS);
+        self.package("rust-docs", &mut manifest.pkg, DOCS_TARGETS);
         self.package("rust-src", &mut manifest.pkg, &["*"]);
         self.package("rls-preview", &mut manifest.pkg, HOSTS);
         self.package("clippy-preview", &mut manifest.pkg, HOSTS);
@@ -402,11 +433,7 @@ impl Builder {
                         Some(p) => p,
                         None => return false,
                     };
-                    let target = match pkg.target.get(&c.target) {
-                        Some(t) => t,
-                        None => return false,
-                    };
-                    target.available
+                    pkg.target.get(&c.target).is_some()
                 };
                 extensions.retain(&has_component);
                 components.retain(&has_component);
@@ -588,6 +615,10 @@ impl Builder {
     }
 
     fn sign(&self, path: &Path) {
+        if !self.should_sign {
+            return;
+        }
+
         let filename = path.file_name().unwrap().to_str().unwrap();
         let asc = self.output.join(format!("{}.asc", filename));
         println!("signing: {:?}", path);