Automatically vendor Cargo deps when building the source tarballs.

This commit is contained in:
Eduard-Mihai Burtescu 2017-02-10 22:59:40 +02:00
parent 42c1ea2915
commit d29f0bc8fa
27 changed files with 106 additions and 36 deletions

View File

@ -47,15 +47,24 @@ matrix:
install: &osx_install_sccache >
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1
- env: >
RUST_CHECK_TARGET=check
RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
SRC=.
os: osx
osx_image: xcode8.2
install: *osx_install_sccache
- env: >
SCRIPT="./x.py test && ./x.py dist"
RUST_CHECK_TARGET=dist
RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended"
SRC=.
DEPLOY=1
os: osx
osx_image: xcode8.2
install: *osx_install_sccache
install: >
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1 && brew uninstall --ignore-dependencies openssl && brew install openssl --universal --without-test
- env: >
RUST_CHECK_TARGET=dist
RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended"

View File

@ -17,7 +17,6 @@ environment:
# MSVC cargotest
- MSYS_BITS: 64
NO_VENDOR: 1
RUST_CHECK_TARGET: check-aux
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc

1
configure vendored
View File

@ -648,6 +648,7 @@ opt rustbuild 1 "use the rust and cargo based build system"
opt codegen-tests 1 "run the src/test/codegen tests"
opt option-checking 1 "complain about unrecognized options in this configure script"
opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
opt locked-deps 0 "force Cargo.lock to be up to date"
opt vendor 0 "enable usage of vendored Rust crates"
opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)"

View File

@ -294,6 +294,8 @@ class RustBuild(object):
raise Exception("no cargo executable found at `%s`" % self.cargo())
args = [self.cargo(), "build", "--manifest-path",
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
if self.use_locked_deps:
args.append("--locked")
if self.use_vendored_sources:
args.append("--frozen")
self.run(args, env)
@ -440,6 +442,9 @@ def main():
rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
'CFG_ENABLE_VENDOR' in rb.config_mk
rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
if os.environ.get('USER') != os.environ['SUDO_USER']:
rb.use_vendored_sources = True

View File

@ -108,8 +108,12 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) {
pub fn tidy(build: &Build, host: &str) {
println!("tidy check ({})", host);
let compiler = Compiler::new(0, host);
build.run(build.tool_cmd(&compiler, "tidy")
.arg(build.src.join("src")));
let mut cmd = build.tool_cmd(&compiler, "tidy");
cmd.arg(build.src.join("src"));
if !build.config.vendor {
cmd.arg("--no-vendor");
}
build.run(&mut cmd);
}
fn testdir(build: &Build, host: &str) -> PathBuf {
@ -643,6 +647,7 @@ pub fn distcheck(build: &Build) {
build.run(&mut cmd);
build.run(Command::new("./configure")
.args(&build.config.configure_args)
.arg("--enable-vendor")
.current_dir(&dir));
build.run(Command::new(build_helper::make(&build.config.build))
.arg("check")

View File

@ -44,6 +44,7 @@ pub struct Config {
pub submodules: bool,
pub compiler_docs: bool,
pub docs: bool,
pub locked_deps: bool,
pub vendor: bool,
pub target_config: HashMap<String, Target>,
pub full_bootstrap: bool,
@ -145,6 +146,7 @@ struct Build {
docs: Option<bool>,
submodules: Option<bool>,
gdb: Option<String>,
locked_deps: Option<bool>,
vendor: Option<bool>,
nodejs: Option<String>,
python: Option<String>,
@ -294,6 +296,7 @@ impl Config {
set(&mut config.compiler_docs, build.compiler_docs);
set(&mut config.docs, build.docs);
set(&mut config.submodules, build.submodules);
set(&mut config.locked_deps, build.locked_deps);
set(&mut config.vendor, build.vendor);
set(&mut config.full_bootstrap, build.full_bootstrap);
set(&mut config.extended, build.extended);
@ -440,6 +443,7 @@ impl Config {
("LOCAL_REBUILD", self.local_rebuild),
("NINJA", self.ninja),
("CODEGEN_TESTS", self.codegen_tests),
("LOCKED_DEPS", self.locked_deps),
("VENDOR", self.vendor),
("FULL_BOOTSTRAP", self.full_bootstrap),
("EXTENDED", self.extended),

View File

@ -108,6 +108,10 @@
# Note that Python 2 is currently required.
#python = "python2.7"
# Force Cargo to check that Cargo.lock describes the precise dependency
# set that all the Cargo.toml files create, instead of updating it.
#locked-deps = false
# Indicate whether the vendored sources are used for Rust dependencies or not
#vendor = false

View File

@ -360,6 +360,8 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
t!(fs::remove_dir_all(&image));
}
const CARGO_VENDOR_VERSION: &'static str = "0.1.4";
/// Creates the `rust-src` installer component and the plain source tarball
pub fn rust_src(build: &Build) {
println!("Dist src");
@ -404,13 +406,6 @@ pub fn rust_src(build: &Build) {
}
}
// If we're inside the vendor directory then we need to preserve
// everything as Cargo's vendoring support tracks all checksums and we
// want to be sure we don't accidentally leave out a file.
if spath.contains("vendor") {
return true
}
let excludes = [
"CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
@ -433,6 +428,29 @@ pub fn rust_src(build: &Build) {
copy(&build.src.join(item), &dst_src.join(item));
}
// Get cargo-vendor installed, if it isn't already.
let mut has_cargo_vendor = false;
let mut cmd = Command::new(&build.cargo);
for line in output(cmd.arg("install").arg("--list")).lines() {
has_cargo_vendor |= line.starts_with("cargo-vendor ");
}
if !has_cargo_vendor {
let mut cmd = Command::new(&build.cargo);
cmd.arg("install")
.arg("--force")
.arg("--debug")
.arg("--vers").arg(CARGO_VENDOR_VERSION)
.arg("cargo-vendor")
.env("RUSTC", &build.rustc);
build.run(&mut cmd);
}
// Vendor all Cargo dependencies
let mut cmd = Command::new(&build.cargo);
cmd.arg("vendor")
.current_dir(&dst_src.join("src"));
build.run(&mut cmd);
// Create source tarball in rust-installer format
let mut cmd = Command::new("sh");
cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))

View File

@ -528,6 +528,9 @@ impl Build {
if self.config.rust_optimize && cmd != "bench" {
cargo.arg("--release");
}
if self.config.locked_deps {
cargo.arg("--locked");
}
if self.config.vendor || self.is_sudo {
cargo.arg("--frozen");
}

View File

@ -16,7 +16,9 @@ RUN dpkg --add-architecture i386 && \
openjdk-9-jre \
sudo \
libstdc++6:i386 \
xz-utils
xz-utils \
libssl-dev \
pkg-config
WORKDIR /android/
ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools

View File

@ -17,7 +17,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gcc-sparc64-linux-gnu \
libc6-dev-sparc64-cross \
bzip2 \
patch
patch \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
texinfo \
wget \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
texinfo \
wget \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
bzip2 \
xz-utils \
wget
wget \
libssl-dev \
pkg-config
COPY build-toolchain.sh /tmp/
RUN /tmp/build-toolchain.sh x86_64

View File

@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gdb \
xz-utils \
g++-mips-linux-gnu \
g++-mipsel-linux-gnu
g++-mipsel-linux-gnu \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gdb \
xz-utils \
g++-mips64-linux-gnuabi64 \
g++-mips64el-linux-gnuabi64
g++-mips64el-linux-gnuabi64 \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
texinfo \
wget \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
texinfo \
wget \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
texinfo \
wget \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -13,13 +13,14 @@ RUN yum upgrade -y && yum install -y \
file \
xz \
which \
pkg-config \
pkgconfig \
wget \
autoconf \
gettext
ENV PATH=/rustroot/bin:$PATH
ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib
ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
WORKDIR /tmp
COPY shared.sh build-binutils.sh /tmp/

View File

@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist
ENV SCRIPT python2.7 ../x.py test

View File

@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
xz-utils \
sudo \
gdb \
patch
patch \
libssl-dev \
pkg-config
WORKDIR /build/
COPY musl-libunwind-patch.patch build-musl.sh /build/

View File

@ -25,4 +25,3 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
ENV RUST_CHECK_TARGET check-aux
ENV NO_VENDOR 1

View File

@ -11,7 +11,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
sudo \
gdb \
xz-utils
xz-utils \
libssl-dev \
pkg-config
ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \

View File

@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist
ENV SCRIPT python2.7 ../x.py test

View File

@ -23,6 +23,7 @@ fi
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
# If we're deploying artifacts then we set the release channel, otherwise if
# we're not deploying then we want to be sure to enable all assertions becauase
@ -47,13 +48,6 @@ else
fi
fi
# We want to enable usage of the `src/vendor` dir as much as possible, but not
# all test suites have all their deps in there (just the main bootstrap) so we
# have the ability to disable this flag
if [ "$NO_VENDOR" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-vendor"
fi
set -ex
$SRC/configure $RUST_CONFIGURE_ARGS

View File

@ -42,6 +42,8 @@ fn main() {
let path = env::args_os().skip(1).next().expect("need an argument");
let path = PathBuf::from(path);
let args: Vec<String> = env::args().skip(1).collect();
let mut bad = false;
bins::check(&path, &mut bad);
style::check(&path, &mut bad);
@ -49,7 +51,9 @@ fn main() {
cargo::check(&path, &mut bad);
features::check(&path, &mut bad);
pal::check(&path, &mut bad);
deps::check(&path, &mut bad);
if !args.iter().any(|s| *s == "--no-vendor") {
deps::check(&path, &mut bad);
}
if bad {
panic!("some tidy checks failed");