Rollup merge of #41958 - malbarbo:docker-share, r=alexcrichton

ci: allows files to be shared by docker images Change `src/ci/docker/run.sh` to allow files to be shared when building docker containers. For while, only android related shell scripts are shared. Others containers can be updated at any time. sccache is installed last, as it is frequently updated this avoids the need to rebuilt the whole container.
2017-05-19 14:16:14 -06:00 · 2017-05-19 14:16:14 -06:00 · 04b07798fc
parent d8d2db264b 9b184c0c17
commit 04b07798fc
34 changed files with 251 additions and 268 deletions
--- a/src/ci/docker/README.md
+++ b/src/ci/docker/README.md
@ -16,6 +16,12 @@ for example:

 Images will output artifacts in an `obj` dir at the root of a repository.

+## Filesystem layout
+
+- Each directory, excluding `scripts` and `disabled`, corresponds to a docker image
+- `scripts` contains files shared by docker images
+- `disabled` contains images that are not build travis
+
 ## Cross toolchains

 A number of these images take quite a long time to compile as they're building
--- a/src/ci/docker/arm-android/Dockerfile
+++ b/src/ci/docker/arm-android/Dockerfile
@ -2,52 +2,44 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-    chmod +x /usr/local/bin/sccache
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip arm 9

-# Install NDK
-COPY install-ndk.sh /tmp
-RUN . /tmp/install-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain arm 9 && \
-    remove_ndk
-
-# Install SDK
+# sdk
 RUN dpkg --add-architecture i386 && \
    apt-get update && \
    apt-get install -y --no-install-recommends \
-  openjdk-9-jre-headless \
-  tzdata \
-  libstdc++6:i386 \
  libgl1-mesa-glx \
-  libpulse0
+  libpulse0 \
+  libstdc++6:i386 \
+  openjdk-9-jre-headless \
+  tzdata

-COPY install-sdk.sh /tmp
-RUN . /tmp/install-sdk.sh && \
-    download_sdk tools_r25.2.5-linux.zip && \
-    download_sysimage armeabi-v7a 18 && \
-    create_avd armeabi-v7a 18
+COPY scripts/android-sdk.sh /scripts/
+RUN . /scripts/android-sdk.sh && \
+    download_and_create_avd tools_r25.2.5-linux.zip armeabi-v7a 18

-# Setup env
+# env
 ENV PATH=$PATH:/android/sdk/tools
 ENV PATH=$PATH:/android/sdk/platform-tools

@ -57,8 +49,12 @@ ENV RUST_CONFIGURE_ARGS \
      --target=$TARGETS \
      --arm-linux-androideabi-ndk=/android/ndk/arm-9

-ENV SCRIPT python2.7 ../x.py test --target $TARGETS --verbose
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS

-# Entrypoint
-COPY start-emulator.sh /android/
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+COPY scripts/android-start-emulator.sh /scripts/
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/scripts/android-start-emulator.sh"]
--- a/src/ci/docker/arm-android/install-ndk.sh
+++ b/src/ci/docker/arm-android/install-ndk.sh
@ -1,35 +0,0 @@
-#!/bin/sh
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-URL=https://dl.google.com/android/repository
-
-download_ndk() {
-    mkdir -p /android/ndk
-    cd /android/ndk
-    curl -O $URL/$1
-    unzip -q $1
-    rm $1
-    mv android-ndk-* ndk
-}
-
-make_standalone_toolchain() {
-    # See https://developer.android.com/ndk/guides/standalone_toolchain.html
-    python2.7 /android/ndk/ndk/build/tools/make_standalone_toolchain.py \
-        --install-dir /android/ndk/$1-$2 \
-        --arch $1 \
-        --api $2
-}
-
-remove_ndk() {
-    rm -rf /android/ndk/ndk
-}
--- a/src/ci/docker/armhf-gnu/Dockerfile
+++ b/src/ci/docker/armhf-gnu/Dockerfile
@ -31,7 +31,7 @@ WORKDIR /build
 # The `vexpress_config` config file was a previously generated config file for
 # the kernel. This file was generated by running `make vexpress_defconfig`
 # followed by `make menuconfig` and then enabling the IPv6 protocol page.
-COPY vexpress_config /build/.config
+COPY armhf-gnu/vexpress_config /build/.config
 RUN curl https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.4.42.tar.xz | \
      tar xJf - && \
      cd /build/linux-4.4.42 && \
@ -63,11 +63,11 @@ RUN curl http://cdimage.ubuntu.com/ubuntu-base/releases/16.04/release/ubuntu-bas

 # Copy over our init script, which starts up our test server and also a few
 # other misc tasks.
-COPY rcS rootfs/etc/init.d/rcS
+COPY armhf-gnu/rcS rootfs/etc/init.d/rcS
 RUN chmod +x rootfs/etc/init.d/rcS

 # Helper to quickly fill the entropy pool in the kernel.
-COPY addentropy.c /tmp/
+COPY armhf-gnu/addentropy.c /tmp/
 RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static

 # TODO: What is this?!
--- a/src/ci/docker/cross/Dockerfile
+++ b/src/ci/docker/cross/Dockerfile
@ -32,10 +32,10 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]

 WORKDIR /tmp

-COPY build-rumprun.sh /tmp/
+COPY cross/build-rumprun.sh /tmp/
 RUN ./build-rumprun.sh

-COPY build-arm-musl.sh /tmp/
+COPY cross/build-arm-musl.sh /tmp/
 RUN ./build-arm-musl.sh

 # originally from
--- a/src/ci/docker/disabled/dist-aarch64-android/Dockerfile
+++ b/src/ci/docker/disabled/dist-aarch64-android/Dockerfile
@ -2,36 +2,30 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain arm64 21 && \
-    remove_ndk
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip arm64 21

+# env
 ENV PATH=$PATH:/android/ndk/arm64-21/bin

 ENV DEP_Z_ROOT=/android/ndk/arm64-21/sysroot/usr/
@ -47,3 +41,10 @@ ENV RUST_CONFIGURE_ARGS \
      --enable-cargo-openssl-static

 ENV SCRIPT python2.7 ../x.py dist --target $HOSTS --host $HOSTS
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
--- a/src/ci/docker/disabled/dist-armv7-android/Dockerfile
+++ b/src/ci/docker/disabled/dist-armv7-android/Dockerfile
@ -2,37 +2,36 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
    download_ndk android-ndk-r13b-linux-x86_64.zip && \
    make_standalone_toolchain arm 9 && \
    make_standalone_toolchain arm 21 && \
    remove_ndk

+RUN chmod 777 /android/ndk && \
+    ln -s /android/ndk/arm-21 /android/ndk/arm
+
+# env
 ENV PATH=$PATH:/android/ndk/arm-9/bin

 ENV DEP_Z_ROOT=/android/ndk/arm-9/sysroot/usr/
@ -54,12 +53,16 @@ ENV RUST_CONFIGURE_ARGS \
 # level 9), the default linker behavior is to generate an error, to allow the
 # build to finish we use --warn-unresolved-symbols. Note that the missing
 # symbols does not affect std, only the compiler (llvm) and cargo (openssl).
-RUN chmod 777 /android/ndk && \
-    ln -s /android/ndk/arm-21 /android/ndk/arm
-
 ENV SCRIPT \
  python2.7 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \
  (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \
    rm /android/ndk/arm && \
    ln -s /android/ndk/arm-9 /android/ndk/arm && \
    python2.7 ../x.py dist --host $HOSTS --target $HOSTS)
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
--- a/src/ci/docker/disabled/dist-i686-android/Dockerfile
+++ b/src/ci/docker/disabled/dist-i686-android/Dockerfile
@ -2,37 +2,36 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
    download_ndk android-ndk-r13b-linux-x86_64.zip && \
    make_standalone_toolchain x86 9 && \
    make_standalone_toolchain x86 21 && \
    remove_ndk

+RUN chmod 777 /android/ndk && \
+    ln -s /android/ndk/x86-21 /android/ndk/x86
+
+# env
 ENV PATH=$PATH:/android/ndk/x86-9/bin

 ENV DEP_Z_ROOT=/android/ndk/x86-9/sysroot/usr/
@ -54,12 +53,16 @@ ENV RUST_CONFIGURE_ARGS \
 # level 9), the default linker behavior is to generate an error, to allow the
 # build to finish we use --warn-unresolved-symbols. Note that the missing
 # symbols does not affect std, only the compiler (llvm) and cargo (openssl).
-RUN chmod 777 /android/ndk && \
-    ln -s /android/ndk/x86-21 /android/ndk/x86
-
 ENV SCRIPT \
  python2.7 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \
  (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \
    rm /android/ndk/x86 && \
    ln -s /android/ndk/x86-9 /android/ndk/x86 && \
    python2.7 ../x.py dist --host $HOSTS --target $HOSTS)
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
--- a/src/ci/docker/disabled/dist-x86_64-android/Dockerfile
+++ b/src/ci/docker/disabled/dist-x86_64-android/Dockerfile
@ -2,36 +2,30 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain x86_64 21 && \
-    remove_ndk
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip x86_64 21

+# env
 ENV PATH=$PATH:/android/ndk/x86_64-21/bin

 ENV DEP_Z_ROOT=/android/ndk/x86_64-21/sysroot/usr/
@ -47,3 +41,10 @@ ENV RUST_CONFIGURE_ARGS \
      --enable-cargo-openssl-static

 ENV SCRIPT python2.7 ../x.py dist --target $HOSTS --host $HOSTS
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
--- a/src/ci/docker/dist-aarch64-linux/Dockerfile
+++ b/src/ci/docker/dist-aarch64-linux/Dockerfile
@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY aarch64-linux-gnu.config build-toolchains.sh /tmp/
+COPY dist-aarch64-linux/aarch64-linux-gnu.config dist-aarch64-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh

 USER root
--- a/src/ci/docker/dist-android/Dockerfile
+++ b/src/ci/docker/dist-android/Dockerfile
@ -2,33 +2,27 @@ FROM ubuntu:16.04

 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
  ca-certificates \
-  python2.7 \
-  git \
  cmake \
-  unzip \
-  sudo \
-  xz-utils \
+  curl \
+  file \
+  g++ \
+  git \
  libssl-dev \
-  pkg-config
+  make \
+  pkg-config \
+  python2.7 \
+  sudo \
+  unzip \
+  xz-utils

-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh

-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-    chmod +x /usr/local/bin/sccache
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-# Install NDK
-COPY install-ndk.sh /tmp
-RUN . /tmp/install-ndk.sh && \
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
    download_ndk android-ndk-r13b-linux-x86_64.zip && \
    make_standalone_toolchain arm 9 && \
    make_standalone_toolchain x86 9 && \
@ -36,6 +30,7 @@ RUN . /tmp/install-ndk.sh && \
    make_standalone_toolchain x86_64 21 && \
    remove_ndk

+# env
 ENV TARGETS=arm-linux-androideabi
 ENV TARGETS=$TARGETS,armv7-linux-androideabi
 ENV TARGETS=$TARGETS,i686-linux-android
@ -52,3 +47,10 @@ ENV RUST_CONFIGURE_ARGS \
      --x86_64-linux-android-ndk=/android/ndk/x86_64-21

 ENV SCRIPT python2.7 ../x.py dist --target $TARGETS
+
+# cache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
--- a/src/ci/docker/dist-android/install-ndk.sh
+++ b/src/ci/docker/dist-android/install-ndk.sh
@ -1,35 +0,0 @@
-#!/bin/sh
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-URL=https://dl.google.com/android/repository
-
-download_ndk() {
-    mkdir -p /android/ndk
-    cd /android/ndk
-    curl -O $URL/$1
-    unzip -q $1
-    rm $1
-    mv android-ndk-* ndk
-}
-
-make_standalone_toolchain() {
-    # See https://developer.android.com/ndk/guides/standalone_toolchain.html
-    python2.7 /android/ndk/ndk/build/tools/make_standalone_toolchain.py \
-        --install-dir /android/ndk/$1-$2 \
-        --arch $1 \
-        --api $2
-}
-
-remove_ndk() {
-    rm -rf /android/ndk/ndk
-}
--- a/src/ci/docker/dist-arm-linux/Dockerfile
+++ b/src/ci/docker/dist-arm-linux/Dockerfile
@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY arm-linux-gnueabi.config build-toolchains.sh /tmp/
+COPY dist-arm-linux/arm-linux-gnueabi.config dist-arm-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh

 USER root
--- a/src/ci/docker/dist-armhf-linux/Dockerfile
+++ b/src/ci/docker/dist-armhf-linux/Dockerfile
@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY arm-linux-gnueabihf.config build-toolchains.sh /tmp/
+COPY dist-armhf-linux/arm-linux-gnueabihf.config dist-armhf-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh

 USER root
--- a/src/ci/docker/dist-armv7-linux/Dockerfile
+++ b/src/ci/docker/dist-armv7-linux/Dockerfile
@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY build-toolchains.sh armv7-linux-gnueabihf.config /tmp/
+COPY dist-armv7-linux/build-toolchains.sh dist-armv7-linux/armv7-linux-gnueabihf.config /tmp/
 RUN ./build-toolchains.sh

 USER root
--- a/src/ci/docker/dist-fuchsia/Dockerfile
+++ b/src/ci/docker/dist-fuchsia/Dockerfile
@ -21,7 +21,7 @@ RUN curl -L https://cmake.org/files/v3.8/cmake-3.8.0-rc1-Linux-x86_64.tar.gz | \
      tar xzf - -C /usr/local --strip-components=1

 WORKDIR /tmp
-COPY shared.sh build-toolchain.sh compiler-rt-dso-handle.patch /tmp/
+COPY dist-fuchsia/shared.sh dist-fuchsia/build-toolchain.sh dist-fuchsia/compiler-rt-dso-handle.patch /tmp/
 RUN /tmp/build-toolchain.sh

 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
--- a/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile
+++ b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile
@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  pkg-config

 WORKDIR /build/
-COPY musl-libunwind-patch.patch build-musl.sh /build/
+COPY dist-i586-gnu-i686-musl/musl-libunwind-patch.patch dist-i586-gnu-i686-musl/build-musl.sh /build/
 RUN sh /build/build-musl.sh && rm -rf /build

 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
--- a/src/ci/docker/dist-i686-freebsd/Dockerfile
+++ b/src/ci/docker/dist-i686-freebsd/Dockerfile
@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  libssl-dev \
  pkg-config

-COPY build-toolchain.sh /tmp/
+COPY dist-i686-freebsd/build-toolchain.sh /tmp/
 RUN /tmp/build-toolchain.sh i686

 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
--- a/src/ci/docker/dist-i686-linux/Dockerfile
+++ b/src/ci/docker/dist-i686-linux/Dockerfile
@ -29,13 +29,13 @@ ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
-COPY shared.sh build-binutils.sh /tmp/
+COPY dist-i686-linux/shared.sh dist-i686-linux/build-binutils.sh /tmp/

 # We need a build of openssl which supports SNI to download artifacts from
 # static.rust-lang.org. This'll be used to link into libcurl below (and used
 # later as well), so build a copy of OpenSSL with dynamic libraries into our
 # generic root.
-COPY build-openssl.sh /tmp/
+COPY dist-i686-linux/build-openssl.sh /tmp/
 RUN ./build-openssl.sh

 # The `curl` binary on CentOS doesn't support SNI which is needed for fetching
@ -44,7 +44,7 @@ RUN ./build-openssl.sh
 #
 # Note that we also disable a bunch of optional features of curl that we don't
 # really need.
-COPY build-curl.sh /tmp/
+COPY dist-i686-linux/build-curl.sh /tmp/
 RUN ./build-curl.sh

 # binutils < 2.22 has a bug where the 32-bit executables it generates
@ -54,26 +54,26 @@ RUN ./build-curl.sh
 RUN ./build-binutils.sh

 # Need a newer version of gcc than centos has to compile LLVM nowadays
-COPY build-gcc.sh /tmp/
+COPY dist-i686-linux/build-gcc.sh /tmp/
 RUN ./build-gcc.sh

 # CentOS 5.5 has Python 2.4 by default, but LLVM needs 2.7+
-COPY build-python.sh /tmp/
+COPY dist-i686-linux/build-python.sh /tmp/
 RUN ./build-python.sh

 # Apparently CentOS 5.5 desn't have `git` in yum, but we're gonna need it for
 # cloning, so download and build it here.
-COPY build-git.sh /tmp/
+COPY dist-i686-linux/build-git.sh /tmp/
 RUN ./build-git.sh

 # libssh2 (a dependency of Cargo) requires cmake 2.8.11 or higher but CentOS
 # only has 2.6.4, so build our own
-COPY build-cmake.sh /tmp/
+COPY dist-i686-linux/build-cmake.sh /tmp/
 RUN ./build-cmake.sh

 # for sanitizers, we need kernel headers files newer than the ones CentOS ships
 # with so we install newer ones here
-COPY build-headers.sh /tmp/
+COPY dist-i686-linux/build-headers.sh /tmp/
 RUN ./build-headers.sh

 RUN curl -Lo /rustroot/dumb-init \
--- a/src/ci/docker/dist-powerpc-linux/Dockerfile
+++ b/src/ci/docker/dist-powerpc-linux/Dockerfile
@ -56,8 +56,8 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY patches/ /tmp/patches/
-COPY powerpc-linux-gnu.config build-powerpc-toolchain.sh /tmp/
+COPY dist-powerpc-linux/patches/ /tmp/patches/
+COPY dist-powerpc-linux/powerpc-linux-gnu.config dist-powerpc-linux/build-powerpc-toolchain.sh /tmp/
 RUN ./build-powerpc-toolchain.sh

 USER root
--- a/src/ci/docker/dist-powerpc64-linux/Dockerfile
+++ b/src/ci/docker/dist-powerpc64-linux/Dockerfile
@ -56,8 +56,8 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY patches/ /tmp/patches/
-COPY shared.sh powerpc64-linux-gnu.config build-powerpc64-toolchain.sh /tmp/
+COPY dist-powerpc64-linux/patches/ /tmp/patches/
+COPY dist-powerpc64-linux/shared.sh dist-powerpc64-linux/powerpc64-linux-gnu.config dist-powerpc64-linux/build-powerpc64-toolchain.sh /tmp/
 RUN ./build-powerpc64-toolchain.sh

 USER root
--- a/src/ci/docker/dist-powerpc64le-linux/Dockerfile
+++ b/src/ci/docker/dist-powerpc64le-linux/Dockerfile
@ -59,7 +59,7 @@ WORKDIR /tmp
 USER root

 RUN apt-get install -y --no-install-recommends rpm2cpio cpio
-COPY shared.sh build-powerpc64le-toolchain.sh /tmp/
+COPY dist-powerpc64le-linux/shared.sh dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /tmp/
 RUN ./build-powerpc64le-toolchain.sh

 RUN curl -o /usr/local/bin/sccache \
--- a/src/ci/docker/dist-s390x-linux/Dockerfile
+++ b/src/ci/docker/dist-s390x-linux/Dockerfile
@ -56,8 +56,8 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY patches/ /tmp/patches/
-COPY s390x-linux-gnu.config build-s390x-toolchain.sh /tmp/
+COPY dist-s390x-linux/patches/ /tmp/patches/
+COPY dist-s390x-linux/s390x-linux-gnu.config dist-s390x-linux/build-s390x-toolchain.sh /tmp/
 RUN ./build-s390x-toolchain.sh

 USER root
--- a/src/ci/docker/dist-x86_64-freebsd/Dockerfile
+++ b/src/ci/docker/dist-x86_64-freebsd/Dockerfile
@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  libssl-dev \
  pkg-config

-COPY build-toolchain.sh /tmp/
+COPY dist-x86_64-freebsd/build-toolchain.sh /tmp/
 RUN /tmp/build-toolchain.sh x86_64

 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
--- a/src/ci/docker/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/dist-x86_64-linux/Dockerfile
@ -29,13 +29,13 @@ ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
-COPY shared.sh build-binutils.sh /tmp/
+COPY dist-x86_64-linux/shared.sh dist-x86_64-linux/build-binutils.sh /tmp/

 # We need a build of openssl which supports SNI to download artifacts from
 # static.rust-lang.org. This'll be used to link into libcurl below (and used
 # later as well), so build a copy of OpenSSL with dynamic libraries into our
 # generic root.
-COPY build-openssl.sh /tmp/
+COPY dist-x86_64-linux/build-openssl.sh /tmp/
 RUN ./build-openssl.sh

 # The `curl` binary on CentOS doesn't support SNI which is needed for fetching
@ -44,7 +44,7 @@ RUN ./build-openssl.sh
 #
 # Note that we also disable a bunch of optional features of curl that we don't
 # really need.
-COPY build-curl.sh /tmp/
+COPY dist-x86_64-linux/build-curl.sh /tmp/
 RUN ./build-curl.sh

 # binutils < 2.22 has a bug where the 32-bit executables it generates
@ -54,26 +54,26 @@ RUN ./build-curl.sh
 RUN ./build-binutils.sh

 # Need a newer version of gcc than centos has to compile LLVM nowadays
-COPY build-gcc.sh /tmp/
+COPY dist-x86_64-linux/build-gcc.sh /tmp/
 RUN ./build-gcc.sh

 # CentOS 5.5 has Python 2.4 by default, but LLVM needs 2.7+
-COPY build-python.sh /tmp/
+COPY dist-x86_64-linux/build-python.sh /tmp/
 RUN ./build-python.sh

 # Apparently CentOS 5.5 desn't have `git` in yum, but we're gonna need it for
 # cloning, so download and build it here.
-COPY build-git.sh /tmp/
+COPY dist-x86_64-linux/build-git.sh /tmp/
 RUN ./build-git.sh

 # libssh2 (a dependency of Cargo) requires cmake 2.8.11 or higher but CentOS
 # only has 2.6.4, so build our own
-COPY build-cmake.sh /tmp/
+COPY dist-x86_64-linux/build-cmake.sh /tmp/
 RUN ./build-cmake.sh

 # for sanitizers, we need kernel headers files newer than the ones CentOS ships
 # with so we install newer ones here
-COPY build-headers.sh /tmp/
+COPY dist-x86_64-linux/build-headers.sh /tmp/
 RUN ./build-headers.sh

 RUN curl -Lo /rustroot/dumb-init \
--- a/src/ci/docker/dist-x86_64-musl/Dockerfile
+++ b/src/ci/docker/dist-x86_64-musl/Dockerfile
@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  pkg-config

 WORKDIR /build/
-COPY build-musl.sh /build/
+COPY dist-x86_64-musl/build-musl.sh /build/
 RUN sh /build/build-musl.sh && rm -rf /build

 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
--- a/src/ci/docker/dist-x86_64-netbsd/Dockerfile
+++ b/src/ci/docker/dist-x86_64-netbsd/Dockerfile
@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp

-COPY build-netbsd-toolchain.sh /tmp/
+COPY dist-x86_64-netbsd/build-netbsd-toolchain.sh /tmp/
 RUN ./build-netbsd-toolchain.sh

 USER root
--- a/src/ci/docker/emscripten/Dockerfile
+++ b/src/ci/docker/emscripten/Dockerfile
@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

 WORKDIR /tmp
-COPY build-emscripten.sh /tmp/
+COPY emscripten/build-emscripten.sh /tmp/
 RUN ./build-emscripten.sh
 ENV PATH=$PATH:/tmp/emsdk_portable
 ENV PATH=$PATH:/tmp/emsdk_portable/clang/tag-e1.37.10/build_tag-e1.37.10_32/bin
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@ -26,7 +26,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
      build \
      --rm \
      -t rust-ci \
-      "$docker_dir/$image"
+      -f "$docker_dir/$image/Dockerfile" \
+      "$docker_dir"
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
    if [ -n "$TRAVIS_OS_NAME" ]; then
        echo Cannot run disabled images on travis!
--- a/src/ci/docker/scripts/android-ndk.sh
+++ b/src/ci/docker/scripts/android-ndk.sh
@ -1,4 +1,3 @@
-#!/bin/sh
 # Copyright 2017 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@ -33,3 +32,9 @@ make_standalone_toolchain() {
 remove_ndk() {
    rm -rf /android/ndk/ndk
 }
+
+download_and_make_toolchain() {
+    download_ndk $1 && \
+    make_standalone_toolchain $2 $3 && \
+    remove_ndk
+}
--- a/src/ci/docker/arm-android/install-sdk.sh
+++ b/src/ci/docker/arm-android/install-sdk.sh
@ -1,4 +1,3 @@
-#!/bin/sh
 # Copyright 2017 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@ -47,3 +46,8 @@ create_avd() {
            --abi $abi
 }

+download_and_create_avd() {
+    download_sdk $1
+    download_sysimage $2 $3
+    create_avd $2 $3
+}
--- a/src/ci/docker/scripts/android-start-emulator.sh
+++ b/src/ci/docker/scripts/android-start-emulator.sh
--- a/src/ci/docker/scripts/dumb-init.sh
+++ b/src/ci/docker/scripts/dumb-init.sh
@ -0,0 +1,15 @@
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb
+dpkg -i dumb-init_*.deb
+rm dumb-init_*.deb
--- a/src/ci/docker/scripts/sccache.sh
+++ b/src/ci/docker/scripts/sccache.sh
@ -0,0 +1,16 @@
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+curl -o /usr/local/bin/sccache \
+  https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl
+
+chmod +x /usr/local/bin/sccache