OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rustbuild: Add sccache support

Browse Source 
This commit adds support for sccache, a ccache-like compiler which works on MSVC
and stores results into an S3 bucket. This also switches over all Travis and
AppVeyor automation to using sccache to ensure a shared and unified cache over
time which can be shared across builders.

The support for sccache manifests as a new `--enable-sccache` option which
instructs us to configure LLVM differently to use a 'sccache' binary instead of
a 'ccache' binary. All docker images for Travis builds are updated to download
Mozilla's tooltool builds of sccache onto various containers and systems.
Additionally a new `rust-lang-ci-sccache` bucket is configured to hold all of
our ccache goodies.
This commit is contained in:
Alex Crichton 2016-12-12 11:36:52 -08:00
parent 0d558d012a
commit 96a5fc76dc
21 changed files with 152 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.travis.yml
View File

@ -30,25 +30,34 @@ matrix:
RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin
SRC=.
os: osx
install: brew install ccache
install: &osx_install_sccache >
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1
- env: >
RUST_CHECK_TARGET=check
RUST_CONFIGURE_ARGS=--target=i686-apple-darwin
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=check
RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --disable-rustbuild
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=
RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
SRC=.
os: osx
install: brew install ccache
install: *osx_install_sccache
env:
global:
- SCCACHE_BUCKET=rust-lang-ci-sccache
- AWS_ACCESS_KEY_ID=AKIAIMX7VLAS3PZAVLUQ
# AWS_SECRET_ACCESS_KEY=...
- secure: "Pixhh0hXDqGCdOyLtGFjli3J2AtDWIpyb2btIrLe956nCBDRutRoMm6rv5DI9sFZN07Mms7VzNNvhc9wCW1y63JAm414d2Co7Ob8kWMZlz9l9t7ACHuktUiis8yr+S4Quq1Vqd6pqi7pf2J++UxC8R/uLeqVrubzr6+X7AbmEFE="
script:
- >
@ -77,5 +86,3 @@ notifications:
cache:
directories:
- $HOME/docker
- $HOME/.ccache
- $HOME/.cargo

12
appveyor.yml
View File

@ -1,4 +1,9 @@
environment:
SCCACHE_BUCKET: rust-lang-ci-sccache
AWS_ACCESS_KEY_ID: AKIAIMX7VLAS3PZAVLUQ
AWS_SECRET_ACCESS_KEY:
secure: 1UkmbiDd15tWtYbMm5O2Uqm0b0Ur8v1MoSlydxl4ojcroPeerRMlUges0l57py8c
SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c
matrix:
# 32/64 bit MSVC
- MSYS_BITS: 64
@ -84,6 +89,13 @@ install:
# Otherwise pull in the MinGW installed on appveyor
- if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH%
# Download and install sccache
- appveyor DownloadFile https://api.pub.build.mozilla.org/tooltool/sha512/%SCCACHE_DIGEST%
- mv %SCCACHE_DIGEST% sccache.tar.bz2
- 7z x -y sccache.tar.bz2 > nul
- 7z x -y sccache.tar > nul
- set PATH=%PATH%;%CD%\sccache2
test_script:
- git submodule update --init
- set SRC=.

21
configure vendored
View File

@ -621,6 +621,7 @@ opt llvm-assertions 0 "build LLVM with assertions"
opt debug-assertions 0 "build with debugging assertions"
opt fast-make 0 "use .gitmodules as timestamp for submodule deps"
opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds"
opt sccache 0 "invoke gcc/clang via sccache to reuse object files between builds"
opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
opt local-rebuild 0 "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version"
opt llvm-static-stdcpp 0 "statically link to libstdc++ for LLVM"
@ -1677,11 +1678,23 @@ do
LLVM_CC_64_ARG1="gcc"
;;
("gcc")
LLVM_CXX_32="g++"
LLVM_CC_32="gcc"
if [ -z "$CFG_ENABLE_SCCACHE" ]; then
LLVM_CXX_32="g++"
LLVM_CC_32="gcc"
LLVM_CXX_64="g++"
LLVM_CC_64="gcc"
LLVM_CXX_64="g++"
LLVM_CC_64="gcc"
else
LLVM_CXX_32="sccache"
LLVM_CC_32="sccache"
LLVM_CXX_32_ARG1="g++"
LLVM_CC_32_ARG1="gcc"
LLVM_CXX_64="sccache"
LLVM_CC_64="sccache"
LLVM_CXX_64_ARG1="g++"
LLVM_CC_64_ARG1="gcc"
fi
;;
(*)

33
src/bootstrap/config.rs
View File

@ -38,7 +38,7 @@ use util::push_exe_path;
/// `src/bootstrap/config.toml.example`.
#[derive(Default)]
pub struct Config {
pub ccache: bool,
pub ccache: Option<String>,
pub ninja: bool,
pub verbose: bool,
pub submodules: bool,
@ -138,7 +138,7 @@ struct Build {
/// TOML representation of how the LLVM build is configured.
#[derive(RustcDecodable, Default)]
struct Llvm {
ccache: Option<bool>,
ccache: Option<StringOrBool>,
ninja: Option<bool>,
assertions: Option<bool>,
optimize: Option<bool>,
@ -147,6 +147,18 @@ struct Llvm {
static_libstdcpp: Option<bool>,
}
#[derive(RustcDecodable)]
enum StringOrBool {
String(String),
Bool(bool),
}
impl Default for StringOrBool {
fn default() -> StringOrBool {
StringOrBool::Bool(false)
}
}
/// TOML representation of how the Rust build is configured.
#[derive(RustcDecodable, Default)]
struct Rust {
@ -247,7 +259,15 @@ impl Config {
set(&mut config.vendor, build.vendor);
if let Some(ref llvm) = toml.llvm {
set(&mut config.ccache, llvm.ccache);
match llvm.ccache {
Some(StringOrBool::String(ref s)) => {
config.ccache = Some(s.to_string())
}
Some(StringOrBool::Bool(true)) => {
config.ccache = Some("ccache".to_string());
}
Some(StringOrBool::Bool(false)) | None => {}
}
set(&mut config.ninja, llvm.ninja);
set(&mut config.llvm_assertions, llvm.assertions);
set(&mut config.llvm_optimize, llvm.optimize);
@ -338,7 +358,6 @@ impl Config {
}
check! {
("CCACHE", self.ccache),
("MANAGE_SUBMODULES", self.submodules),
("COMPILER_DOCS", self.compiler_docs),
("DOCS", self.docs),
@ -475,6 +494,12 @@ impl Config {
let path = parse_configure_path(value);
self.python = Some(path);
}
"CFG_ENABLE_CCACHE" if value == "1" => {
self.ccache = Some("ccache".to_string());
}
"CFG_ENABLE_SCCACHE" if value == "1" => {
self.ccache = Some("sccache".to_string());
}
_ => {}
}
}

2
src/bootstrap/config.toml.example
View File

@ -25,6 +25,8 @@
# Indicates whether ccache is used when building LLVM
#ccache = false
# or alternatively ...
#ccache = "/path/to/ccache"
# If an external LLVM root is specified, we automatically check the version by
# default to make sure it's within the range that we're expecting, but setting

6
src/bootstrap/native.rs
View File

@ -109,10 +109,10 @@ pub fn llvm(build: &Build, target: &str) {
// MSVC handles compiler business itself
if !target.contains("msvc") {
if build.config.ccache {
cfg.define("CMAKE_C_COMPILER", "ccache")
if let Some(ref ccache) = build.config.ccache {
cfg.define("CMAKE_C_COMPILER", ccache)
.define("CMAKE_C_COMPILER_ARG1", build.cc(target))
.define("CMAKE_CXX_COMPILER", "ccache")
.define("CMAKE_CXX_COMPILER", ccache)
.define("CMAKE_CXX_COMPILER_ARG1", build.cxx(target));
} else {
cfg.define("CMAKE_C_COMPILER", build.cc(target))

4
src/bootstrap/sanity.rs
View File

@ -223,4 +223,8 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
if build.lldb_version.is_some() {
build.lldb_python_dir = run(Command::new("lldb").arg("-P")).ok();
}
if let Some(ref s) = build.config.ccache {
need_cmd(s.as_ref());
}
}

7
src/ci/docker/arm-android/Dockerfile
View File

@ -16,7 +16,8 @@ RUN dpkg --add-architecture i386 && \
expect \
openjdk-9-jre \
sudo \
libstdc++6:i386
libstdc++6:i386 \
xz-utils
WORKDIR /android/
ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
@ -33,6 +34,10 @@ COPY start-emulator.sh /android/
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV TARGETS=arm-linux-androideabi
ENV TARGETS=$TARGETS,i686-linux-android
ENV TARGETS=$TARGETS,aarch64-linux-android

7
src/ci/docker/cross/Dockerfile
View File

@ -21,7 +21,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
gcc-s390x-linux-gnu libc6-dev-s390x-cross
gcc-s390x-linux-gnu libc6-dev-s390x-cross \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/i686-gnu-nopt/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/i686-gnu/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

6
src/ci/docker/run.sh
View File

@ -25,7 +25,6 @@ docker \
-t rust-ci \
"`dirname "$script"`/$image"
mkdir -p $HOME/.ccache
mkdir -p $HOME/.cargo
mkdir -p $root_dir/obj
@ -35,8 +34,9 @@ exec docker \
--volume "$root_dir/obj:/checkout/obj" \
--workdir /checkout/obj \
--env SRC=/checkout \
--env CCACHE_DIR=/ccache \
--volume "$HOME/.ccache:/ccache" \
--env SCCACHE_BUCKET=$SCCACHE_BUCKET \
--env AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
--env CARGO_HOME=/cargo \
--env LOCAL_USER_ID=`id -u` \
--volume "$HOME/.cargo:/cargo" \

4
src/ci/docker/x86_64-freebsd/Dockerfile
View File

@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV \
AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc

7
src/ci/docker/x86_64-gnu-cargotest/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
libssl-dev \
sudo
sudo \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-debug/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile
View File

@ -14,7 +14,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gdb \
llvm-3.7-tools \
libedit-dev \
zlib1g-dev
zlib1g-dev \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-make/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu-nopt/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

7
src/ci/docker/x86_64-gnu/Dockerfile
View File

@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
ccache \
sudo \
gdb
gdb \
xz-utils
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \

4
src/ci/docker/x86_64-musl/Dockerfile
View File

@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
tar xJf - -C /usr/local/bin --strip-components=1
ENV RUST_CONFIGURE_ARGS \
--target=x86_64-unknown-linux-musl \
--musl-root-x86_64=/musl-x86_64

6
src/ci/run.sh
View File

@ -26,17 +26,13 @@ if [ "$NO_VENDOR" = "" ]; then
ENABLE_VENDOR=--enable-vendor
fi
if [ "$NO_CCACHE" = "" ]; then
ENABLE_CCACHE=--enable-ccache
fi
set -ex
$SRC/configure \
--disable-manage-submodules \
--enable-debug-assertions \
--enable-quiet-tests \
$ENABLE_CCACHE \
--enable-sccache \
$ENABLE_VENDOR \
$ENABLE_LLVM_ASSERTIONS \
$RUST_CONFIGURE_ARGS