Rollup merge of #32926 - caipre:rustbuild-verify-download, r=alexcrichton

rustbuild: Verify sha256 of downloaded tarballs

Here's a quick first pass at this.

I don't use Python often enough to claim that this is totally Pythonic. I've left off some (almost certainly unnecessary) error handling regarding opening and processing files. The whole tarball is read into memory to calculate the hash, but the file isn't *so* large so that should be fine. I don't care for the output from `raise RuntimeError`, but that's how `run()` does it so I'm following precedent.

Tested by manually changing the value of `expected`, and by modifying the tarball then forcing `rustc_out_of_date()`. Both cases tripped the error.

Closes https://github.com/rust-lang/rust/issues/32902

src/bootstrap/bootstrap.py

@@ -10,6 +10,7 @@
 
 import argparse
 import contextlib
+import hashlib
 import os
 import shutil
 import subprocess
@@ -18,13 +19,29 @@ import tarfile
 
 def get(url, path, verbose=False):
     print("downloading " + url)
-    # see http://serverfault.com/questions/301128/how-to-download
+    sha_url = url + ".sha256"
-    if sys.platform == 'win32':
+    sha_path = path + ".sha256"
-        run(["PowerShell.exe", "/nologo", "-Command",
+    for _url, _path in ((url, path), (sha_url, sha_path)):
-             "(New-Object System.Net.WebClient).DownloadFile('" + url +
+        # see http://serverfault.com/questions/301128/how-to-download
-                "', '" + path + "')"], verbose=verbose)
+        if sys.platform == 'win32':
-    else:
+            run(["PowerShell.exe", "/nologo", "-Command",
-        run(["curl", "-o", path, url], verbose=verbose)
+                 "(New-Object System.Net.WebClient)"
+                 ".DownloadFile('{}', '{}')".format(_url, _path)],
+                verbose=verbose)
+        else:
+            run(["curl", "-o", _path, _url], verbose=verbose)
+    print("verifying " + path)
+    with open(path, "rb") as f:
+        found = hashlib.sha256(f.read()).hexdigest()
+    with open(sha_path, "r") as f:
+        expected, _ = f.readline().split()
+    if found != expected:
+        err = ("invalid checksum:\n"
+               "    found:    {}\n"
+               "    expected: {}".format(found, expected))
+        if verbose:
+            raise RuntimeError(err)
+        sys.exit(err)
 
 def unpack(tarball, dst, verbose=False, match=None):
     print("extracting " + tarball)
@@ -57,9 +74,10 @@ def run(args, verbose=False):
     ret = subprocess.Popen(args)
     code = ret.wait()
     if code != 0:
-        if not verbose:
+        err = "failed to run: " + ' '.join(args)
-            print("failed to run: " + ' '.join(args))
+        if verbose:
-        raise RuntimeError("failed to run command")
+            raise RuntimeError(err)
+        sys.exit(err)
 
 class RustBuild:
     def download_rust_nightly(self):
@@ -210,7 +228,10 @@ class RustBuild:
             if sys.platform == 'win32':
                 return 'x86_64-pc-windows-msvc'
             else:
-                raise
+                err = "uname not found"
+                if self.verbose:
+                    raise Exception(err)
+                sys.exit(err)
 
         # Darwin's `uname -s` lies and always returns i386. We have to use
         # sysctl instead.
@@ -253,7 +274,10 @@ class RustBuild:
                 cputype = 'x86_64'
             ostype = 'pc-windows-gnu'
         else:
-            raise ValueError("unknown OS type: " + ostype)
+            err = "unknown OS type: " + ostype
+            if self.verbose:
+                raise ValueError(err)
+            sys.exit(err)
 
         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
             cputype = 'i686'
@@ -269,7 +293,10 @@ class RustBuild:
         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
             cputype = 'x86_64'
         else:
-            raise ValueError("unknown cpu type: " + cputype)
+            err = "unknown cpu type: " + cputype
+            if self.verbose:
+                raise ValueError(err)
+            sys.exit(err)
 
         return cputype + '-' + ostype