OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Auto merge of #38359 - alexcrichton:sccache, r=brson

Browse Source 
rustbuild: Add sccache support

This commit adds support for sccache, a ccache-like compiler which works on MSVC
and stores results into an S3 bucket. This also switches over all Travis and
AppVeyor automation to using sccache to ensure a shared and unified cache over
time which can be shared across builders.

The support for sccache manifests as a new `--enable-sccache` option which
instructs us to configure LLVM differently to use a 'sccache' binary instead of
a 'ccache' binary. All docker images for Travis builds are updated to download
Mozilla's tooltool builds of sccache onto various containers and systems.
Additionally a new `rust-lang-ci-sccache` bucket is configured to hold all of
our ccache goodies.

---

Note that this does not currently change Windows [due to previously written up issues](https://github.com/rust-lang/rust/issues/38119#issuecomment-266631585). Despite that, however, I was curious to get timings for the builds on Travis to see what ranges we're working with. As a result, this is a WIP PR I'm using to gauge build times and such.
This commit is contained in:
bors 2016-12-16 07:26:41 +00:00
commit d250169cb5
21 changed files with 152 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.travis.yml
View File

@ -30,25 +30,34 @@ matrix:
RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin
SRC=.
os: osx
install: brew install ccache
install: &osx_install_sccache >
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1
- env: >
RUST_CHECK_TARGET=check
RUST_CONFIGURE_ARGS=--target=i686-apple-darwin
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=check
RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --disable-rustbuild
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=
RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
env:
global:
- SCCACHE_BUCKET=rust-lang-ci-sccache
- AWS_ACCESS_KEY_ID=AKIAIMX7VLAS3PZAVLUQ
# AWS_SECRET_ACCESS_KEY=...
- secure: "Pixhh0hXDqGCdOyLtGFjli3J2AtDWIpyb2btIrLe956nCBDRutRoMm6rv5DI9sFZN07Mms7VzNNvhc9wCW1y63JAm414d2Co7Ob8kWMZlz9l9t7ACHuktUiis8yr+S4Quq1Vqd6pqi7pf2J++UxC8R/uLeqVrubzr6+X7AbmEFE="
script:
- >
@ -77,5 +86,3 @@ notifications:
cache:
directories:
- $HOME/docker
- $HOME/.ccache
- $HOME/.cargo

12
appveyor.yml
View File

@ -1,4 +1,9 @@
environment:
SCCACHE_BUCKET: rust-lang-ci-sccache
AWS_ACCESS_KEY_ID: AKIAIMX7VLAS3PZAVLUQ
AWS_SECRET_ACCESS_KEY:
secure: 1UkmbiDd15tWtYbMm5O2Uqm0b0Ur8v1MoSlydxl4ojcroPeerRMlUges0l57py8c
SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c
matrix:
# 32/64 bit MSVC
- MSYS_BITS: 64
@ -84,6 +89,13 @@ install:
# Otherwise pull in the MinGW installed on appveyor
- if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH%
# Download and install sccache
- appveyor DownloadFile https://api.pub.build.mozilla.org/tooltool/sha512/%SCCACHE_DIGEST%
- mv %SCCACHE_DIGEST% sccache.tar.bz2
- 7z x -y sccache.tar.bz2 > nul
- 7z x -y sccache.tar > nul
- set PATH=%PATH%;%CD%\sccache2
test_script:
- git submodule update --init
- set SRC=.

21
configure vendored
View File

@ -621,6 +621,7 @@ opt llvm-assertions 0 "build LLVM with assertions"
opt debug-assertions 0 "build with debugging assertions"
opt fast-make 0 "use .gitmodules as timestamp for submodule deps"
opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds"
opt sccache 0 "invoke gcc/clang via sccache to reuse object files between builds"
opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
opt local-rebuild 0 "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version"
opt llvm-static-stdcpp 0 "statically link to libstdc++ for LLVM"
@ -1677,11 +1678,23 @@ do
LLVM_CC_64_ARG1="gcc"
;;
("gcc")
LLVM_CXX_32="g++"
LLVM_CC_32="gcc"
if [ -z "$CFG_ENABLE_SCCACHE" ]; then
LLVM_CXX_32="g++"
LLVM_CC_32="gcc"
LLVM_CXX_64="g++"
LLVM_CC_64="gcc"
LLVM_CXX_64="g++"
LLVM_CC_64="gcc"
else
LLVM_CXX_32="sccache"
LLVM_CC_32="sccache"
LLVM_CXX_32_ARG1="g++"
LLVM_CC_32_ARG1="gcc"
LLVM_CXX_64="sccache"
LLVM_CC_64="sccache"
LLVM_CXX_64_ARG1="g++"
LLVM_CC_64_ARG1="gcc"
fi
;;
(*)

33
src/bootstrap/config.rs
View File

@ -38,7 +38,7 @@ use util::push_exe_path;
/// `src/bootstrap/config.toml.example`.
#[derive(Default)]
pub struct Config {
pub ccache: bool,
pub ccache: Option<String>,
pub ninja: bool,
pub verbose: bool,
pub submodules: bool,
@ -138,7 +138,7 @@ struct Build {
/// TOML representation of how the LLVM build is configured.
#[derive(RustcDecodable, Default)]
struct Llvm {
ccache: Option<bool>,
ccache: Option<StringOrBool>,
ninja: Option<bool>,
assertions: Option<bool>,
optimize: Option<bool>,
@ -147,6 +147,18 @@ struct Llvm {
static_libstdcpp: Option<bool>,
}
#[derive(RustcDecodable)]
enum StringOrBool {
String(String),
Bool(bool),
}
impl Default for StringOrBool {
fn default() -> StringOrBool {
StringOrBool::Bool(false)
}
}
/// TOML representation of how the Rust build is configured.
#[derive(RustcDecodable, Default)]
struct Rust {
@ -247,7 +259,15 @@ impl Config {
set(&mut config.vendor, build.vendor);
if let Some(ref llvm) = toml.llvm {
set(&mut config.ccache, llvm.ccache);
match llvm.ccache {
Some(StringOrBool::String(ref s)) => {
config.ccache = Some(s.to_string())
}
Some(StringOrBool::Bool(true)) => {
config.ccache = Some("ccache".to_string());
}
Some(StringOrBool::Bool(false)) | None => {}
}
set(&mut config.ninja, llvm.ninja);
set(&mut config.llvm_assertions, llvm.assertions);
set(&mut config.llvm_optimize, llvm.optimize);
@ -338,7 +358,6 @@ impl Config {
}
check! {
("CCACHE", self.ccache),
("MANAGE_SUBMODULES", self.submodules),
("COMPILER_DOCS", self.compiler_docs),
("DOCS", self.docs),
@ -475,6 +494,12 @@ impl Config {
let path = parse_configure_path(value);
self.python = Some(path);
}
"CFG_ENABLE_CCACHE" if value == "1" => {
self.ccache = Some("ccache".to_string());
}
"CFG_ENABLE_SCCACHE" if value == "1" => {
self.ccache = Some("sccache".to_string());
}
_ => {}
}
}

2
src/bootstrap/config.toml.example
View File

@ -25,6 +25,8 @@
# Indicates whether ccache is used when building LLVM
#ccache = false
# or alternatively ...
#ccache = "/path/to/ccache"
# If an external LLVM root is specified, we automatically check the version by
# default to make sure it's within the range that we're expecting, but setting

6
src/bootstrap/native.rs
View File

@ -109,10 +109,10 @@ pub fn llvm(build: &Build, target: &str) {
// MSVC handles compiler business itself
if !target.contains("msvc") {
if build.config.ccache {
cfg.define("CMAKE_C_COMPILER", "ccache")
if let Some(ref ccache) = build.config.ccache {
cfg.define("CMAKE_C_COMPILER", ccache)
.define("CMAKE_C_COMPILER_ARG1", build.cc(target))
.define("CMAKE_CXX_COMPILER", "ccache")
.define("CMAKE_CXX_COMPILER", ccache)
.define("CMAKE_CXX_COMPILER_ARG1", build.cxx(target));
} else {
cfg.define("CMAKE_C_COMPILER", build.cc(target))

4
src/bootstrap/sanity.rs
View File

@ -223,4 +223,8 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
if build.lldb_version.is_some() {
build.lldb_python_dir = run(Command::new("lldb").arg("-P")).ok();
}
if let Some(ref s) = build.config.ccache {
need_cmd(s.as_ref());
}
}

7
src/ci/docker/arm-android/Dockerfile
View File

@ -16,7 +16,8 @@ RUN dpkg --add-architecture i386 && \
expect \
openjdk-9-jre \
sudo \
libstdc++6:i386
libstdc++6:i386 \
xz-utils
WORKDIR /android/
ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
@ -33,6 +34,10 @@ COPY start-emulator.sh /android/
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV TARGETS=arm-linux-androideabi
ENV TARGETS=$TARGETS,i686-linux-android
ENV TARGETS=$TARGETS,aarch64-linux-android

7
src/ci/docker/cross/Dockerfile
View File

@ -21,7 +21,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
gcc-s390x-linux-gnu libc6-dev-s390x-cross
gcc-s390x-linux-gnu libc6-dev-s390x-cross \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/i686-gnu-nopt/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/i686-gnu/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

6
src/ci/docker/run.sh
View File

@ -25,7 +25,6 @@ docker \
-t rust-ci \
"`dirname "$script"`/$image"
mkdir -p $HOME/.ccache
mkdir -p $HOME/.cargo
mkdir -p $root_dir/obj
@ -35,8 +34,9 @@ exec docker \
--volume "$root_dir/obj:/checkout/obj" \
--workdir /checkout/obj \
--env SRC=/checkout \
--env CCACHE_DIR=/ccache \
--volume "$HOME/.ccache:/ccache" \
--env SCCACHE_BUCKET=$SCCACHE_BUCKET \
--env AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
--env CARGO_HOME=/cargo \
--env LOCAL_USER_ID=`id -u` \
--volume "$HOME/.cargo:/cargo" \

4
src/ci/docker/x86_64-freebsd/Dockerfile
View File

@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV \
AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc

7
src/ci/docker/x86_64-gnu-cargotest/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
libssl-dev \
sudo
sudo \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-debug/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile
View File

@ -14,7 +14,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gdb \
llvm-3.7-tools \
libedit-dev \
zlib1g-dev
zlib1g-dev \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-make/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-nopt/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

4
src/ci/docker/x86_64-musl/Dockerfile
View File

@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV RUST_CONFIGURE_ARGS \
--target=x86_64-unknown-linux-musl \
--musl-root-x86_64=/musl-x86_64

6
src/ci/run.sh
View File

@ -26,17 +26,13 @@ if [ "$NO_VENDOR" = "" ]; then
ENABLE_VENDOR=--enable-vendor
fi
if [ "$NO_CCACHE" = "" ]; then
ENABLE_CCACHE=--enable-ccache
fi
set -ex
$SRC/configure \
--disable-manage-submodules \
--enable-debug-assertions \
--enable-quiet-tests \
$ENABLE_CCACHE \
--enable-sccache \
$ENABLE_VENDOR \
$ENABLE_LLVM_ASSERTIONS \
$RUST_CONFIGURE_ARGS