From f44801c5ee71861c67977992d69d14bfb1a702d3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 Mar 2017 19:46:44 -0800 Subject: [PATCH] travis: Split the linux-tested-targets builder Travis only gives us 30GB disk space and we don't currently have an option to increase that. Each musl target generates "hello world" binaries of about 3.5MB in size, and we're testing two targets in the same image. We have around 3k run-pass tests and 2 musl targets which works out to around 20GB. That's dangerously close to the limit and is causing PRs to bounce. This PR splits up the builder in two, one for x86_64 musl and the other for i686. Hopefully that'll keep us under the disk limit. Closes #40359 --- .travis.yml | 3 +- .../Dockerfile | 8 +--- .../build-musl.sh | 15 ------- .../musl-libunwind-patch.patch | 0 src/ci/docker/dist-x86_64-musl/Dockerfile | 45 +++++++++++++++++++ src/ci/docker/dist-x86_64-musl/build-musl.sh | 38 ++++++++++++++++ 6 files changed, 87 insertions(+), 22 deletions(-) rename src/ci/docker/{linux-tested-targets => dist-i586-gnu-i686-musl}/Dockerfile (79%) rename src/ci/docker/{linux-tested-targets => dist-i586-gnu-i686-musl}/build-musl.sh (80%) rename src/ci/docker/{linux-tested-targets => dist-i586-gnu-i686-musl}/musl-libunwind-patch.patch (100%) create mode 100644 src/ci/docker/dist-x86_64-musl/Dockerfile create mode 100644 src/ci/docker/dist-x86_64-musl/build-musl.sh diff --git a/.travis.yml b/.travis.yml index 2f12408242f..fa6c9cf0748 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,17 +15,18 @@ matrix: - env: IMAGE=arm-android - env: IMAGE=armhf-gnu - env: IMAGE=cross DEPLOY=1 - - env: IMAGE=linux-tested-targets DEPLOY=1 - env: IMAGE=dist-android DEPLOY=1 - env: IMAGE=dist-arm-linux DEPLOY=1 - env: IMAGE=dist-armv7-aarch64-linux DEPLOY=1 - env: IMAGE=dist-freebsd DEPLOY=1 + - env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1 - env: IMAGE=dist-mips-linux DEPLOY=1 - env: IMAGE=dist-mips64-linux DEPLOY=1 - env: IMAGE=dist-powerpc-linux DEPLOY=1 - env: IMAGE=dist-powerpc64-linux DEPLOY=1 - env: IMAGE=dist-s390x-linux-netbsd DEPLOY=1 - env: IMAGE=dist-x86-linux DEPLOY=1 + - env: IMAGE=dist-x86_64-musl DEPLOY=1 - env: IMAGE=emscripten - env: IMAGE=i686-gnu - env: IMAGE=i686-gnu-nopt diff --git a/src/ci/docker/linux-tested-targets/Dockerfile b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile similarity index 79% rename from src/ci/docker/linux-tested-targets/Dockerfile rename to src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile index a7060cd55fe..0897be80cc2 100644 --- a/src/ci/docker/linux-tested-targets/Dockerfile +++ b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile @@ -30,8 +30,7 @@ RUN curl -o /usr/local/bin/sccache \ chmod +x /usr/local/bin/sccache ENV RUST_CONFIGURE_ARGS \ - --target=x86_64-unknown-linux-musl,i686-unknown-linux-musl,i586-unknown-linux-gnu \ - --musl-root-x86_64=/musl-x86_64 \ + --target=i686-unknown-linux-musl,i586-unknown-linux-gnu \ --musl-root-i686=/musl-i686 # Newer binutils broke things on some vms/distros (i.e., linking against @@ -39,16 +38,13 @@ ENV RUST_CONFIGURE_ARGS \ # way to produce "super compatible" binaries. # # See: https://github.com/rust-lang/rust/issues/34978 -ENV CFLAGS_i686_unknown_linux_musl=-Wa,-mrelax-relocations=no \ - CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no +ENV CFLAGS_i686_unknown_linux_musl=-Wa,-mrelax-relocations=no ENV SCRIPT \ python2.7 ../x.py test \ - --target x86_64-unknown-linux-musl \ --target i686-unknown-linux-musl \ --target i586-unknown-linux-gnu \ && \ python2.7 ../x.py dist \ - --target x86_64-unknown-linux-musl \ --target i686-unknown-linux-musl \ --target i586-unknown-linux-gnu diff --git a/src/ci/docker/linux-tested-targets/build-musl.sh b/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh similarity index 80% rename from src/ci/docker/linux-tested-targets/build-musl.sh rename to src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh index ce62c392241..a50a25c7913 100644 --- a/src/ci/docker/linux-tested-targets/build-musl.sh +++ b/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh @@ -18,11 +18,6 @@ export CXXFLAGS="-Wa,-mrelax-relocations=no" MUSL=musl-1.1.14 curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf - cd $MUSL -./configure --prefix=/musl-x86_64 --disable-shared -make -j10 -make install -make clean -# for i686 CFLAGS="$CFLAGS -m32" ./configure --prefix=/musl-i686 --disable-shared --target=i686 make -j10 make install @@ -50,16 +45,6 @@ cd .. mkdir libunwind-build cd libunwind-build -cmake ../libunwind-release_37 -DLLVM_PATH=/build/llvm-release_37 \ - -DLIBUNWIND_ENABLE_SHARED=0 -make -j10 -cp lib/libunwind.a /musl-x86_64/lib - -# (Note: the next cmake call doesn't fully override the previous cached one, so remove the cached -# configuration manually. IOW, if don't do this or call make clean we'll end up building libunwind -# for x86_64 again) -rm -rf * -# for i686 CFLAGS="$CFLAGS -m32" CXXFLAGS="$CXXFLAGS -m32" cmake ../libunwind-release_37 \ -DLLVM_PATH=/build/llvm-release_37 \ -DLIBUNWIND_ENABLE_SHARED=0 diff --git a/src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch b/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch similarity index 100% rename from src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch rename to src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch diff --git a/src/ci/docker/dist-x86_64-musl/Dockerfile b/src/ci/docker/dist-x86_64-musl/Dockerfile new file mode 100644 index 00000000000..5877404ece1 --- /dev/null +++ b/src/ci/docker/dist-x86_64-musl/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + make \ + file \ + curl \ + ca-certificates \ + python2.7 \ + git \ + cmake \ + xz-utils \ + sudo \ + gdb \ + patch \ + libssl-dev \ + pkg-config + +WORKDIR /build/ +COPY build-musl.sh /build/ +RUN sh /build/build-musl.sh && rm -rf /build + +RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ + dpkg -i dumb-init_*.deb && \ + rm dumb-init_*.deb +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +RUN curl -o /usr/local/bin/sccache \ + https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-25-sccache-x86_64-unknown-linux-musl && \ + chmod +x /usr/local/bin/sccache + +ENV RUST_CONFIGURE_ARGS \ + --target=x86_64-unknown-linux-musl \ + --musl-root-x86_64=/musl-x86_64 + +# Newer binutils broke things on some vms/distros (i.e., linking against +# unknown relocs disabled by the following flag), so we need to go out of our +# way to produce "super compatible" binaries. +# +# See: https://github.com/rust-lang/rust/issues/34978 +ENV CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no + +ENV SCRIPT \ + python2.7 ../x.py test --target x86_64-unknown-linux-musl && \ + python2.7 ../x.py dist --target x86_64-unknown-linux-musl diff --git a/src/ci/docker/dist-x86_64-musl/build-musl.sh b/src/ci/docker/dist-x86_64-musl/build-musl.sh new file mode 100644 index 00000000000..86bb259c854 --- /dev/null +++ b/src/ci/docker/dist-x86_64-musl/build-musl.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well +export CFLAGS="-fPIC -Wa,-mrelax-relocations=no" +export CXXFLAGS="-Wa,-mrelax-relocations=no" + +MUSL=musl-1.1.14 +curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf - +cd $MUSL +./configure --prefix=/musl-x86_64 --disable-shared +make -j10 +make install + +cd .. +rm -rf $MUSL + +# To build MUSL we're going to need a libunwind lying around, so acquire that +# here and build it. +curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf - +curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf - + +mkdir libunwind-build +cd libunwind-build +cmake ../libunwind-release_37 -DLLVM_PATH=/build/llvm-release_37 \ + -DLIBUNWIND_ENABLE_SHARED=0 +make -j10 +cp lib/libunwind.a /musl-x86_64/lib