bootstrap: Major refactoring

This commit includes the following:

* Fix syntax errors in Python 3
* Include more docstrings in classes, methods, and functions
* Include unit tests using `unittest`
* Merge implementation of `{rustc,cargo}_out_of_date`
* Merge implementation of `RustBuild.{cargo,rustc}`
* Remove unnecessary source code
* Move all the attributes defined outside of `__init__`
* Remove remaining `%s` from print function
* Remove `WindowsError` reference on non-windows systems
* Rename some variables to be more explicit avoid their meaning
* Run bootstrap tests in the CI process
* Remove non-pythonic getters
* Remove duplicate code in `download_stage0` method
* Reduce the number of branches in `build_bootstrap` method
* Re-raise exception when we cannot execute uname in non-windows systems
* Avoid long lines

src/bootstrap/bootstrap.py

@@ -37,12 +37,12 @@ def get(url, path, verbose=False):
         if os.path.exists(path):
             if verify(path, sha_path, False):
                 if verbose:
-                    print("using already-download file " + path)
+                    print("using already-download file", path)
                 return
             else:
                 if verbose:
-                    print("ignoring already-download file " +
+                    print("ignoring already-download file",
-                          path + " due to failed verification")
+                          path, "due to failed verification")
                 os.unlink(path)
         download(temp_path, url, True, verbose)
         if not verify(temp_path, sha_path, verbose):
@@ -59,12 +59,12 @@ def delete_if_present(path, verbose):
     """Remove the given file if present"""
     if os.path.isfile(path):
         if verbose:
-            print("removing " + path)
+            print("removing", path)
         os.unlink(path)
 
 
 def download(path, url, probably_big, verbose):
-    for x in range(0, 4):
+    for _ in range(0, 4):
         try:
             _download(path, url, probably_big, verbose, True)
             return
@@ -96,7 +96,7 @@ def _download(path, url, probably_big, verbose, exception):
 def verify(path, sha_path, verbose):
     """Check if the sha256 sum of the given path is valid"""
     if verbose:
-        print("verifying " + path)
+        print("verifying", path)
     with open(path, "rb") as source:
         found = hashlib.sha256(source.read()).hexdigest()
     with open(sha_path, "r") as sha256sum:
@@ -111,29 +111,30 @@ def verify(path, sha_path, verbose):
 
 def unpack(tarball, dst, verbose=False, match=None):
     """Unpack the given tarball file"""
-    print("extracting " + tarball)
+    print("extracting", tarball)
     fname = os.path.basename(tarball).replace(".tar.gz", "")
     with contextlib.closing(tarfile.open(tarball)) as tar:
-        for p in tar.getnames():
+        for member in tar.getnames():
-            if "/" not in p:
+            if "/" not in member:
                 continue
-            name = p.replace(fname + "/", "", 1)
+            name = member.replace(fname + "/", "", 1)
             if match is not None and not name.startswith(match):
                 continue
             name = name[len(match) + 1:]
 
-            fp = os.path.join(dst, name)
+            dst_path = os.path.join(dst, name)
             if verbose:
-                print("  extracting " + p)
+                print("  extracting", member)
-            tar.extract(p, dst)
+            tar.extract(member, dst)
-            tp = os.path.join(dst, p)
+            src_path = os.path.join(dst, member)
-            if os.path.isdir(tp) and os.path.exists(fp):
+            if os.path.isdir(src_path) and os.path.exists(dst_path):
                 continue
-            shutil.move(tp, fp)
+            shutil.move(src_path, dst_path)
     shutil.rmtree(os.path.join(dst, fname))
 
 
 def run(args, verbose=False, exception=False, **kwargs):
+    """Run a child program in a new process"""
     if verbose:
         print("running: " + ' '.join(args))
     sys.stdout.flush()
@@ -149,97 +150,118 @@ def run(args, verbose=False, exception=False, **kwargs):
 
 
 def stage0_data(rust_root):
+    """Build a dictionary from stage0.txt"""
     nightlies = os.path.join(rust_root, "src/stage0.txt")
-    data = {}
     with open(nightlies, 'r') as nightlies:
-        for line in nightlies:
+        lines = [line.rstrip() for line in nightlies
-            line = line.rstrip()  # Strip newline character, '\n'
+                 if not line.startswith("#")]
-            if line.startswith("#") or line == '':
+        return dict([line.split(": ", 1) for line in lines if line])
-                continue
-            a, b = line.split(": ", 1)
-            data[a] = b
-    return data
 
 
 def format_build_time(duration):
+    """Return a nicer format for build time
+
+    >>> format_build_time('300')
+    '0:05:00'
+    """
     return str(datetime.timedelta(seconds=int(duration)))
 
 
 class RustBuild(object):
+    """Provide all the methods required to build Rust"""
+    def __init__(self):
+        self.cargo_channel = ''
+        self.date = ''
+        self._download_url = 'https://static.rust-lang.org'
+        self.rustc_channel = ''
+        self.build = ''
+        self.build_dir = os.path.join(os.getcwd(), "build")
+        self.clean = False
+        self.config_mk = ''
+        self.config_toml = ''
+        self.printed = False
+        self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
+        self.use_locked_deps = ''
+        self.use_vendored_sources = ''
+        self.verbose = False
 
     def download_stage0(self):
-        cache_dst = os.path.join(self.build_dir, "cache")
+        """Fetch the build system for Rust, written in Rust
-        rustc_cache = os.path.join(cache_dst, self.stage0_date())
-        if not os.path.exists(rustc_cache):
-            os.makedirs(rustc_cache)
 
-        rustc_channel = self.stage0_rustc_channel()
+        This method will build a cache directory, then it will fetch the
-        cargo_channel = self.stage0_cargo_channel()
+        tarball which has the stage0 compiler used to then bootstrap the Rust
+        compiler itself.
+
+        Each downloaded tarball is extracted, after that, the script
+        will move all the content to the right place.
+        """
+        rustc_channel = self.rustc_channel
+        cargo_channel = self.cargo_channel
 
         if self.rustc().startswith(self.bin_root()) and \
-                (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
+                (not os.path.exists(self.rustc()) or
-            self.print_what_it_means_to_bootstrap()
+                 self.program_out_of_date(self.rustc_stamp())):
+            self.print_what_bootstrap_means()
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
             filename = "rust-std-{}-{}.tar.gz".format(
                 rustc_channel, self.build)
-            url = self._download_url + "/dist/" + self.stage0_date()
+            pattern = "rust-std-{}".format(self.build)
-            tarball = os.path.join(rustc_cache, filename)
+            self._download_stage0_helper(filename, pattern)
-            if not os.path.exists(tarball):
-                get("{}/{}".format(url, filename),
-                    tarball, verbose=self.verbose)
-            unpack(tarball, self.bin_root(),
-                   match="rust-std-" + self.build,
-                   verbose=self.verbose)
 
             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
-            url = self._download_url + "/dist/" + self.stage0_date()
+            self._download_stage0_helper(filename, "rustc")
-            tarball = os.path.join(rustc_cache, filename)
+            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
-            if not os.path.exists(tarball):
+            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
-                get("{}/{}".format(url, filename),
+            with open(self.rustc_stamp(), 'w') as rust_stamp:
-                    tarball, verbose=self.verbose)
+                rust_stamp.write(self.date)
-            unpack(tarball, self.bin_root(),
-                   match="rustc", verbose=self.verbose)
-            self.fix_executable(self.bin_root() + "/bin/rustc")
-            self.fix_executable(self.bin_root() + "/bin/rustdoc")
-            with open(self.rustc_stamp(), 'w') as f:
-                f.write(self.stage0_date())
 
             if "pc-windows-gnu" in self.build:
                 filename = "rust-mingw-{}-{}.tar.gz".format(
                     rustc_channel, self.build)
-                url = self._download_url + "/dist/" + self.stage0_date()
+                self._download_stage0_helper(filename, "rust-mingw")
-                tarball = os.path.join(rustc_cache, filename)
-                if not os.path.exists(tarball):
-                    get("{}/{}".format(url, filename),
-                        tarball, verbose=self.verbose)
-                unpack(tarball, self.bin_root(),
-                       match="rust-mingw", verbose=self.verbose)
 
         if self.cargo().startswith(self.bin_root()) and \
-                (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
+                (not os.path.exists(self.cargo()) or
-            self.print_what_it_means_to_bootstrap()
+                 self.program_out_of_date(self.cargo_stamp())):
+            self.print_what_bootstrap_means()
             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
-            url = self._download_url + "/dist/" + self.stage0_date()
+            self._download_stage0_helper(filename, "cargo")
-            tarball = os.path.join(rustc_cache, filename)
+            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
-            if not os.path.exists(tarball):
+            with open(self.cargo_stamp(), 'w') as cargo_stamp:
-                get("{}/{}".format(url, filename),
+                cargo_stamp.write(self.date)
-                    tarball, verbose=self.verbose)
-            unpack(tarball, self.bin_root(),
-                   match="cargo", verbose=self.verbose)
-            self.fix_executable(self.bin_root() + "/bin/cargo")
-            with open(self.cargo_stamp(), 'w') as f:
-                f.write(self.stage0_date())
 
-    def fix_executable(self, fname):
+    def _download_stage0_helper(self, filename, pattern):
-        # If we're on NixOS we need to change the path to the dynamic loader
+        cache_dst = os.path.join(self.build_dir, "cache")
+        rustc_cache = os.path.join(cache_dst, self.date)
+        if not os.path.exists(rustc_cache):
+            os.makedirs(rustc_cache)
 
+        url = "{}/dist/{}".format(self._download_url, self.date)
+        tarball = os.path.join(rustc_cache, filename)
+        if not os.path.exists(tarball):
+            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
+        unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
+
+    @staticmethod
+    def fix_executable(fname):
+        """Modifies the interpreter section of 'fname' to fix the dynamic linker
+
+        This method is only required on NixOS and uses the PatchELF utility to
+        change the dynamic linker of ELF executables.
+
+        Please see https://nixos.org/patchelf.html for more information
+        """
         default_encoding = sys.getdefaultencoding()
         try:
             ostype = subprocess.check_output(
                 ['uname', '-s']).strip().decode(default_encoding)
-        except (subprocess.CalledProcessError, WindowsError):
+        except subprocess.CalledProcessError:
             return
+        except OSError as reason:
+            if getattr(reason, 'winerror', None) is not None:
+                return
+            raise reason
 
         if ostype != "Linux":
             return
@@ -257,8 +279,8 @@ class RustBuild(object):
             interpreter = subprocess.check_output(
                 ["patchelf", "--print-interpreter", fname])
             interpreter = interpreter.strip().decode(default_encoding)
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError as reason:
-            print("warning: failed to call patchelf: %s" % e)
+            print("warning: failed to call patchelf:", reason)
             return
 
         loader = interpreter.split("/")[-1]
@@ -267,8 +289,8 @@ class RustBuild(object):
             ldd_output = subprocess.check_output(
                 ['ldd', '/run/current-system/sw/bin/sh'])
             ldd_output = ldd_output.strip().decode(default_encoding)
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError as reason:
-            print("warning: unable to call ldd: %s" % e)
+            print("warning: unable to call ldd:", reason)
             return
 
         for line in ldd_output.splitlines():
@@ -285,45 +307,66 @@ class RustBuild(object):
         try:
             subprocess.check_output(
                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError as reason:
-            print("warning: failed to call patchelf: %s" % e)
+            print("warning: failed to call patchelf:", reason)
             return
 
-    def stage0_date(self):
-        return self._date
-
-    def stage0_rustc_channel(self):
-        return self._rustc_channel
-
-    def stage0_cargo_channel(self):
-        return self._cargo_channel
-
     def rustc_stamp(self):
-        """Return the path for .rustc-stamp"""
+        """Return the path for .rustc-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
+        True
+        """
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
     def cargo_stamp(self):
-        """Return the path for .cargo-stamp"""
+        """Return the path for .cargo-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
+        True
+        """
         return os.path.join(self.bin_root(), '.cargo-stamp')
 
-    def rustc_out_of_date(self):
+    def program_out_of_date(self, stamp_path):
-        """Check if rustc is out of date"""
+        """Check if the given program stamp is out of date"""
-        if not os.path.exists(self.rustc_stamp()) or self.clean:
+        if not os.path.exists(stamp_path) or self.clean:
             return True
-        with open(self.rustc_stamp(), 'r') as f:
+        with open(stamp_path, 'r') as stamp:
-            return self.stage0_date() != f.read()
+            return self.date != stamp.read()
-
-    def cargo_out_of_date(self):
-        """Check if cargo is out of date"""
-        if not os.path.exists(self.cargo_stamp()) or self.clean:
-            return True
-        with open(self.cargo_stamp(), 'r') as f:
-            return self.stage0_date() != f.read()
 
     def bin_root(self):
+        """Return the binary root directory
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.bin_root() == os.path.join("build", "stage0")
+        True
+
+        When the 'build' property is given should be a nested directory:
+
+        >>> rb.build = "devel"
+        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
+        True
+        """
         return os.path.join(self.build_dir, self.build, "stage0")
 
     def get_toml(self, key):
+        """Returns the value of the given key in config.toml, otherwise returns None
+
+        >>> rb = RustBuild()
+        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
+        >>> rb.get_toml("key2")
+        'value2'
+
+        If the key does not exists, the result is None:
+
+        >>> rb.get_toml("key3") == None
+        True
+        """
         for line in self.config_toml.splitlines():
             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
             if match is not None:
@@ -332,6 +375,18 @@ class RustBuild(object):
         return None
 
     def get_mk(self, key):
+        """Returns the value of the given key in config.mk, otherwise returns None
+
+        >>> rb = RustBuild()
+        >>> rb.config_mk = 'key := value\\n'
+        >>> rb.get_mk('key')
+        'value'
+
+        If the key does not exists, the result is None:
+
+        >>> rb.get_mk('does_not_exists') == None
+        True
+        """
         for line in iter(self.config_mk.splitlines()):
             if line.startswith(key + ' '):
                 var = line[line.find(':=') + 2:].strip()
@@ -340,36 +395,64 @@ class RustBuild(object):
         return None
 
     def cargo(self):
-        config = self.get_toml('cargo')
+        """Return config path for cargo"""
-        if config:
+        return self.program_config('cargo')
-            return config
-        config = self.get_mk('CFG_LOCAL_RUST_ROOT')
-        if config:
-            return config + '/bin/cargo' + self.exe_suffix()
-        return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 
     def rustc(self):
-        config = self.get_toml('rustc')
+        """Return config path for rustc"""
+        return self.program_config('rustc')
+
+    def program_config(self, program):
+        """Return config path for the given program
+
+        >>> rb = RustBuild()
+        >>> rb.config_toml = 'rustc = "rustc"\\n'
+        >>> rb.config_mk = 'CFG_LOCAL_RUST_ROOT := /tmp/rust\\n'
+        >>> rb.program_config('rustc')
+        'rustc'
+        >>> cargo_path = rb.program_config('cargo')
+        >>> cargo_path.rstrip(".exe") == os.path.join("/tmp/rust",
+        ... "bin", "cargo")
+        True
+        >>> rb.config_toml = ''
+        >>> rb.config_mk = ''
+        >>> cargo_path = rb.program_config('cargo')
+        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
+        ... "bin", "cargo")
+        True
+        """
+        config = self.get_toml(program)
         if config:
             return config
         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
         if config:
-            return config + '/bin/rustc' + self.exe_suffix()
+            return os.path.join(config, "bin", "{}{}".format(
-        return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
+                program, self.exe_suffix()))
+        return os.path.join(self.bin_root(), "bin", "{}{}".format(
+            program, self.exe_suffix()))
 
-    def get_string(self, line):
+    @staticmethod
+    def get_string(line):
+        """Return the value between double quotes
+
+        >>> RustBuild.get_string('    "devel"   ')
+        'devel'
+        """
         start = line.find('"')
         if start == -1:
             return None
         end = start + 1 + line[start + 1:].find('"')
         return line[start + 1:end]
 
-    def exe_suffix(self):
+    @staticmethod
+    def exe_suffix():
+        """Return a suffix for executables"""
         if sys.platform == 'win32':
             return '.exe'
         return ''
 
-    def print_what_it_means_to_bootstrap(self):
+    def print_what_bootstrap_means(self):
+        """Prints more information about the build system"""
         if hasattr(self, 'printed'):
             return
         self.printed = True
@@ -386,10 +469,19 @@ class RustBuild(object):
         print('      src/bootstrap/README.md before the download finishes')
 
     def bootstrap_binary(self):
-        return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
+        """Return the path of the boostrap binary
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
+        ... "debug", "bootstrap")
+        True
+        """
+        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 
     def build_bootstrap(self):
-        self.print_what_it_means_to_bootstrap()
+        """Build bootstrap"""
+        self.print_what_bootstrap_means()
         build_dir = os.path.join(self.build_dir, "bootstrap")
         if self.clean and os.path.exists(build_dir):
             shutil.rmtree(build_dir)
@@ -409,7 +501,8 @@ class RustBuild(object):
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
             os.pathsep + env["PATH"]
         if not os.path.isfile(self.cargo()):
-            raise Exception("no cargo executable found at `%s`" % self.cargo())
+            raise Exception("no cargo executable found at `{}`".format(
+                self.cargo()))
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
         if self.verbose:
@@ -423,6 +516,7 @@ class RustBuild(object):
         run(args, env=env, verbose=self.verbose)
 
     def build_triple(self):
+        """Build triple as in LLVM"""
         default_encoding = sys.getdefaultencoding()
         config = self.get_toml('build')
         if config:
@@ -445,23 +539,26 @@ class RustBuild(object):
 
         # The goal here is to come up with the same triple as LLVM would,
         # at least for the subset of platforms we're willing to target.
-        if ostype == 'Linux':
+        ostype_mapper = {
+            'Bitrig': 'unknown-bitrig',
+            'Darwin': 'apple-darwin',
+            'DragonFly': 'unknown-dragonfly',
+            'FreeBSD': 'unknown-freebsd',
+            'Haiku': 'unknown-haiku',
+            'NetBSD': 'unknown-netbsd',
+            'OpenBSD': 'unknown-openbsd'
+        }
+
+        # Consider the direct transformation first and then the special cases
+        if ostype in ostype_mapper:
+            ostype = ostype_mapper[ostype]
+        elif ostype == 'Linux':
             os_from_sp = subprocess.check_output(
                 ['uname', '-o']).strip().decode(default_encoding)
             if os_from_sp == 'Android':
                 ostype = 'linux-android'
             else:
                 ostype = 'unknown-linux-gnu'
-        elif ostype == 'FreeBSD':
-            ostype = 'unknown-freebsd'
-        elif ostype == 'DragonFly':
-            ostype = 'unknown-dragonfly'
-        elif ostype == 'Bitrig':
-            ostype = 'unknown-bitrig'
-        elif ostype == 'OpenBSD':
-            ostype = 'unknown-openbsd'
-        elif ostype == 'NetBSD':
-            ostype = 'unknown-netbsd'
         elif ostype == 'SunOS':
             ostype = 'sun-solaris'
             # On Solaris, uname -m will return a machine classification instead
@@ -477,10 +574,6 @@ class RustBuild(object):
                 if self.verbose:
                     raise Exception(err)
                 sys.exit(err)
-        elif ostype == 'Darwin':
-            ostype = 'apple-darwin'
-        elif ostype == 'Haiku':
-            ostype = 'unknown-haiku'
         elif ostype.startswith('MINGW'):
             # msys' `uname` does not print gcc configuration, but prints msys
             # configuration. so we cannot believe `uname -m`:
@@ -499,13 +592,36 @@ class RustBuild(object):
                 cputype = 'x86_64'
             ostype = 'pc-windows-gnu'
         else:
-            err = "unknown OS type: " + ostype
+            err = "unknown OS type: {}".format(ostype)
             if self.verbose:
                 raise ValueError(err)
             sys.exit(err)
 
-        if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
+        cputype_mapper = {
-            cputype = 'i686'
+            'BePC': 'i686',
+            'aarch64': 'aarch64',
+            'amd64': 'x86_64',
+            'arm64': 'aarch64',
+            'i386': 'i686',
+            'i486': 'i686',
+            'i686': 'i686',
+            'i786': 'i686',
+            'powerpc': 'powerpc',
+            'powerpc64': 'powerpc64',
+            'powerpc64le': 'powerpc64le',
+            'ppc': 'powerpc',
+            'ppc64': 'powerpc64',
+            'ppc64le': 'powerpc64le',
+            's390x': 's390x',
+            'x64': 'x86_64',
+            'x86': 'i686',
+            'x86-64': 'x86_64',
+            'x86_64': 'x86_64'
+        }
+
+        # Consider the direct transformation first and then the special cases
+        if cputype in cputype_mapper:
+            cputype = cputype_mapper[cputype]
         elif cputype in {'xscale', 'arm'}:
             cputype = 'arm'
             if ostype == 'linux-android':
@@ -522,40 +638,26 @@ class RustBuild(object):
                 ostype = 'linux-androideabi'
             else:
                 ostype += 'eabihf'
-        elif cputype in {'aarch64', 'arm64'}:
-            cputype = 'aarch64'
         elif cputype == 'mips':
             if sys.byteorder == 'big':
                 cputype = 'mips'
             elif sys.byteorder == 'little':
                 cputype = 'mipsel'
             else:
-                raise ValueError('unknown byteorder: ' + sys.byteorder)
+                raise ValueError("unknown byteorder: {}".format(sys.byteorder))
         elif cputype == 'mips64':
             if sys.byteorder == 'big':
                 cputype = 'mips64'
             elif sys.byteorder == 'little':
                 cputype = 'mips64el'
             else:
-                raise ValueError('unknown byteorder: ' + sys.byteorder)
+                raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
             # only the n64 ABI is supported, indicate it
             ostype += 'abi64'
-        elif cputype in {'powerpc', 'ppc'}:
-            cputype = 'powerpc'
-        elif cputype in {'powerpc64', 'ppc64'}:
-            cputype = 'powerpc64'
-        elif cputype in {'powerpc64le', 'ppc64le'}:
-            cputype = 'powerpc64le'
         elif cputype == 'sparcv9':
             pass
-        elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
-            cputype = 'x86_64'
-        elif cputype == 's390x':
-            cputype = 's390x'
-        elif cputype == 'BePC':
-            cputype = 'i686'
         else:
-            err = "unknown cpu type: " + cputype
+            err = "unknown cpu type: {}".format(cputype)
             if self.verbose:
                 raise ValueError(err)
             sys.exit(err)
@@ -563,6 +665,7 @@ class RustBuild(object):
         return "{}-{}".format(cputype, ostype)
 
     def update_submodules(self):
+        """Update submodules"""
         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
                 self.get_toml('submodules') == "false" or \
                 self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
@@ -592,8 +695,13 @@ class RustBuild(object):
              "clean", "-qdfx"],
             cwd=self.rust_root, verbose=self.verbose)
 
+    def set_dev_environment(self):
+        """Set download URL for development environment"""
+        self._download_url = 'https://dev-static.rust-lang.org'
+
 
 def bootstrap():
+    """Configure, fetch, build and run the initial bootstrap"""
     parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
     parser.add_argument('--build')
@@ -604,107 +712,103 @@ def bootstrap():
     args, _ = parser.parse_known_args(args)
 
     # Configure initial bootstrap
-    rb = RustBuild()
+    build = RustBuild()
-    rb.config_toml = ''
+    build.verbose = args.verbose
-    rb.config_mk = ''
+    build.clean = args.clean
-    rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
-    rb.build_dir = os.path.join(os.getcwd(), "build")
-    rb.verbose = args.verbose
-    rb.clean = args.clean
 
     try:
         with open(args.config or 'config.toml') as config:
-            rb.config_toml = config.read()
+            build.config_toml = config.read()
     except:
         pass
     try:
-        rb.config_mk = open('config.mk').read()
+        build.config_mk = open('config.mk').read()
     except:
         pass
 
-    if '\nverbose = 2' in rb.config_toml:
+    if '\nverbose = 2' in build.config_toml:
-        rb.verbose = 2
+        build.verbose = 2
-    elif '\nverbose = 1' in rb.config_toml:
+    elif '\nverbose = 1' in build.config_toml:
-        rb.verbose = 1
+        build.verbose = 1
 
-    rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
+    build.use_vendored_sources = '\nvendor = true' in build.config_toml or \
-                              'CFG_ENABLE_VENDOR' in rb.config_mk
+                                 'CFG_ENABLE_VENDOR' in build.config_mk
 
-    rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
+    build.use_locked_deps = '\nlocked-deps = true' in build.config_toml or \
-                         'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
+                            'CFG_ENABLE_LOCKED_DEPS' in build.config_mk
 
-    if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
+    if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
         if os.environ.get('USER') != os.environ['SUDO_USER']:
-            rb.use_vendored_sources = True
+            build.use_vendored_sources = True
             print('info: looks like you are running this command under `sudo`')
             print('      and so in order to preserve your $HOME this will now')
             print('      use vendored sources by default. Note that if this')
             print('      does not work you should run a normal build first')
             print('      before running a command like `sudo make install`')
 
-    if rb.use_vendored_sources:
+    if build.use_vendored_sources:
         if not os.path.exists('.cargo'):
             os.makedirs('.cargo')
-        with open('.cargo/config', 'w') as f:
+        with open('.cargo/config', 'w') as cargo_config:
-            f.write("""
+            cargo_config.write("""
                 [source.crates-io]
                 replace-with = 'vendored-sources'
                 registry = 'https://example.com'
 
                 [source.vendored-sources]
                 directory = '{}/src/vendor'
-            """.format(rb.rust_root))
+            """.format(build.rust_root))
     else:
         if os.path.exists('.cargo'):
             shutil.rmtree('.cargo')
 
-    data = stage0_data(rb.rust_root)
+    data = stage0_data(build.rust_root)
-    rb._date = data['date']
+    build.date = data['date']
-    rb._rustc_channel = data['rustc']
+    build.rustc_channel = data['rustc']
-    rb._cargo_channel = data['cargo']
+    build.cargo_channel = data['cargo']
-    if 'dev' in data:
-        rb._download_url = 'https://dev-static.rust-lang.org'
-    else:
-        rb._download_url = 'https://static.rust-lang.org'
 
-    rb.update_submodules()
+    if 'dev' in data:
+        build.set_dev_environment()
+
+    build.update_submodules()
 
     # Fetch/build the bootstrap
-    rb.build = args.build or rb.build_triple()
+    build.build = args.build or build.build_triple()
-    rb.download_stage0()
+    build.download_stage0()
     sys.stdout.flush()
-    rb.build_bootstrap()
+    build.build_bootstrap()
     sys.stdout.flush()
 
     # Run the bootstrap
-    args = [rb.bootstrap_binary()]
+    args = [build.bootstrap_binary()]
     args.extend(sys.argv[1:])
     env = os.environ.copy()
-    env["BUILD"] = rb.build
+    env["BUILD"] = build.build
-    env["SRC"] = rb.rust_root
+    env["SRC"] = build.rust_root
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     env["BOOTSTRAP_PYTHON"] = sys.executable
-    run(args, env=env, verbose=rb.verbose)
+    run(args, env=env, verbose=build.verbose)
 
 
 def main():
+    """Entry point for the bootstrap process"""
     start_time = time()
     help_triggered = (
         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
     try:
         bootstrap()
         if not help_triggered:
-            print("Build completed successfully in %s" %
+            print("Build completed successfully in {}".format(
-                  format_build_time(time() - start_time))
+                format_build_time(time() - start_time)))
-    except (SystemExit, KeyboardInterrupt) as e:
+    except (SystemExit, KeyboardInterrupt) as error:
-        if hasattr(e, 'code') and isinstance(e.code, int):
+        if hasattr(error, 'code') and isinstance(error.code, int):
-            exit_code = e.code
+            exit_code = error.code
         else:
             exit_code = 1
-            print(e)
+            print(error)
         if not help_triggered:
-            print("Build completed unsuccessfully in %s" %
+            print("Build completed unsuccessfully in {}".format(
-                  format_build_time(time() - start_time))
+                format_build_time(time() - start_time)))
         sys.exit(exit_code)
 
 

src/bootstrap/bootstrap_test.py

@@ -0,0 +1,114 @@
+# Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+"""Bootstrap tests"""
+
+import os
+import doctest
+import unittest
+import tempfile
+import hashlib
+
+from shutil import rmtree
+
+import bootstrap
+
+
+class Stage0DataTestCase(unittest.TestCase):
+    """Test Case for stage0_data"""
+    def setUp(self):
+        self.rust_root = tempfile.mkdtemp()
+        os.mkdir(os.path.join(self.rust_root, "src"))
+        with open(os.path.join(self.rust_root, "src",
+                               "stage0.txt"), "w") as stage0:
+            stage0.write("#ignore\n\ndate: 2017-06-15\nrustc: beta\ncargo: beta")
+
+    def tearDown(self):
+        rmtree(self.rust_root)
+
+    def test_stage0_data(self):
+        """Extract data from stage0.txt"""
+        expected = {"date": "2017-06-15", "rustc": "beta", "cargo": "beta"}
+        data = bootstrap.stage0_data(self.rust_root)
+        self.assertDictEqual(data, expected)
+
+
+class VerifyTestCase(unittest.TestCase):
+    """Test Case for verify"""
+    def setUp(self):
+        self.container = tempfile.mkdtemp()
+        self.src = os.path.join(self.container, "src.txt")
+        self.sums = os.path.join(self.container, "sums")
+        self.bad_src = os.path.join(self.container, "bad.txt")
+        content = "Hello world"
+
+        with open(self.src, "w") as src:
+            src.write(content)
+        with open(self.sums, "w") as sums:
+            sums.write(hashlib.sha256(content.encode("utf-8")).hexdigest())
+        with open(self.bad_src, "w") as bad:
+            bad.write("Hello!")
+
+    def tearDown(self):
+        rmtree(self.container)
+
+    def test_valid_file(self):
+        """Check if the sha256 sum of the given file is valid"""
+        self.assertTrue(bootstrap.verify(self.src, self.sums, False))
+
+    def test_invalid_file(self):
+        """Should verify that the file is invalid"""
+        self.assertFalse(bootstrap.verify(self.bad_src, self.sums, False))
+
+
+class ProgramOutOfDate(unittest.TestCase):
+    """Test if a program is out of date"""
+    def setUp(self):
+        self.container = tempfile.mkdtemp()
+        os.mkdir(os.path.join(self.container, "stage0"))
+        self.build = bootstrap.RustBuild()
+        self.build.date = "2017-06-15"
+        self.build.build_dir = self.container
+        self.rustc_stamp_path = os.path.join(self.container, "stage0",
+                                             ".rustc-stamp")
+
+    def tearDown(self):
+        rmtree(self.container)
+
+    def test_stamp_path_does_not_exists(self):
+        """Return True when the stamp file does not exists"""
+        if os.path.exists(self.rustc_stamp_path):
+            os.unlink(self.rustc_stamp_path)
+        self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path))
+
+    def test_dates_are_different(self):
+        """Return True when the dates are different"""
+        with open(self.rustc_stamp_path, "w") as rustc_stamp:
+            rustc_stamp.write("2017-06-14")
+        self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path))
+
+    def test_same_dates(self):
+        """Return False both dates match"""
+        with open(self.rustc_stamp_path, "w") as rustc_stamp:
+            rustc_stamp.write("2017-06-15")
+        self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path))
+
+
+if __name__ == '__main__':
+    SUITE = unittest.TestSuite()
+    TEST_LOADER = unittest.TestLoader()
+    SUITE.addTest(doctest.DocTestSuite(bootstrap))
+    SUITE.addTests([
+        TEST_LOADER.loadTestsFromTestCase(Stage0DataTestCase),
+        TEST_LOADER.loadTestsFromTestCase(VerifyTestCase),
+        TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)])
+
+    RUNNER = unittest.TextTestRunner(verbosity=2)
+    RUNNER.run(SUITE)

src/bootstrap/mk/Makefile.in

@@ -64,6 +64,8 @@ check-aux:
 		src/test/run-pass-fulldeps/pretty \
 		src/test/run-fail-fulldeps/pretty \
 		$(BOOTSTRAP_ARGS)
+check-bootstrap:
+	$(Q)$(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap_test.py
 dist:
 	$(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
 distcheck:

src/ci/run.sh

@@ -74,6 +74,12 @@ retry make prepare
 travis_fold end make-prepare
 travis_time_finish
 
+travis_fold start check-bootstrap
+travis_time_start
+make check-bootstrap
+travis_fold end check-bootstrap
+travis_time_finish
+
 if [ "$TRAVIS_OS_NAME" = "osx" ]; then
     ncpus=$(sysctl -n hw.ncpu)
 else