OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update Cargo, build curl/OpenSSL statically via features 

In addition to to updating Cargo's submodule and Cargo's dependencies,
this also updates Cargo's build to build OpenSSL statically into Cargo
as well as libcurl unconditionally. This removes OpenSSL build logic
from the bootstrap code, and otherwise requests that even on OSX we
build curl statically.
This commit is contained in:
Alex Crichton 2018-10-08 10:39:09 -07:00
parent d5418764ad
commit 4f661c016f
19 changed files with 285 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.toml.example
View File

@ -182,10 +182,9 @@
# Build the profiler runtime
#profiler = false
# Indicates whether the OpenSSL linked into Cargo will be statically linked or
# not. If static linkage is specified then the build system will download a
# known-good version of OpenSSL, compile it, and link it to Cargo.
#openssl-static = false
# Indicates whether the native libraries linked into Cargo will be statically
# linked or not.
#cargo-native-static = false
# Run the build with low priority, by setting the process group's "nice" value
# to +10 on Unix platforms, and by using a "low priority" job object on Windows.

380
src/Cargo.lock
View File

File diff suppressed because it is too large Load Diff

3
src/bootstrap/check.rs
View File

@ -224,7 +224,8 @@ impl Step for Rustdoc {
target,
"check",
"src/tools/rustdoc",
SourceType::InTree);
SourceType::InTree,
&[]);
let _folder = builder.fold_output(|| format!("stage{}-rustdoc", compiler.stage));
println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);

6
src/bootstrap/config.rs
View File

@ -149,7 +149,7 @@ pub struct Config {
pub nodejs: Option<PathBuf>,
pub gdb: Option<PathBuf>,
pub python: Option<PathBuf>,
pub openssl_static: bool,
pub cargo_native_static: bool,
pub configure_args: Vec<String>,
// These are either the stage0 downloaded binaries or the locally installed ones.
@ -221,7 +221,7 @@ struct Build {
verbose: Option<usize>,
sanitizers: Option<bool>,
profiler: Option<bool>,
openssl_static: Option<bool>,
cargo_native_static: Option<bool>,
configure_args: Option<Vec<String>>,
local_rebuild: Option<bool>,
print_step_timings: Option<bool>,
@ -474,7 +474,7 @@ impl Config {
set(&mut config.verbose, build.verbose);
set(&mut config.sanitizers, build.sanitizers);
set(&mut config.profiler, build.profiler);
set(&mut config.openssl_static, build.openssl_static);
set(&mut config.cargo_native_static, build.cargo_native_static);
set(&mut config.configure_args, build.configure_args);
set(&mut config.local_rebuild, build.local_rebuild);
set(&mut config.print_step_timings, build.print_step_timings);

2
src/bootstrap/configure.py
View File

@ -64,7 +64,7 @@ o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date")
o("vendor", "build.vendor", "enable usage of vendored Rust crates")
o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan)")
o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball")
o("cargo-openssl-static", "build.openssl-static", "static openssl in cargo")
o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo")
o("profiler", "build.profiler", "build the profiler runtime")
o("emscripten", None, "compile the emscripten backend as well as LLVM")
o("full-tools", None, "enable all tools")

7
src/bootstrap/dist.rs
View File

@ -31,7 +31,6 @@ use channel;
use util::{libdir, is_dylib, exe};
use builder::{Builder, RunConfig, ShouldRun, Step};
use compile;
use native;
use tool::{self, Tool};
use cache::{INTERNER, Interned};
use time;
@ -990,12 +989,6 @@ impl Step for PlainSourceTarball {
.arg("--debug")
.arg("--vers").arg(CARGO_VENDOR_VERSION)
.arg("cargo-vendor");
if let Some(dir) = builder.openssl_install_dir(builder.config.build) {
builder.ensure(native::Openssl {
target: builder.config.build,
});
cmd.env("OPENSSL_DIR", dir);
}
builder.run(&mut cmd);
}

1
src/bootstrap/doc.rs
View File

@ -805,6 +805,7 @@ impl Step for Rustdoc {
"doc",
"src/tools/rustdoc",
SourceType::InTree,
&[]
);
cargo.env("RUSTDOCFLAGS", "--document-private-items");

19
src/bootstrap/lib.rs
View File

@ -926,25 +926,6 @@ impl Build {
(self.hosts.iter().any(|h| *h == target) || target == self.build)
}
/// Returns the directory that OpenSSL artifacts are compiled into if
/// configured to do so.
fn openssl_dir(&self, target: Interned<String>) -> Option<PathBuf> {
// OpenSSL not used on Windows
if target.contains("windows") {
None
} else if self.config.openssl_static {
Some(self.out.join(&*target).join("openssl"))
} else {
None
}
}
/// Returns the directory that OpenSSL artifacts are installed into if
/// configured as such.
fn openssl_install_dir(&self, target: Interned<String>) -> Option<PathBuf> {
self.openssl_dir(target).map(|p| p.join("install"))
}
/// Given `num` in the form "a.b.c" return a "release string" which
/// describes the release version number.
///

186
src/bootstrap/native.rs
View File

@ -531,189 +531,3 @@ impl Step for TestHelpers {
.compile("rust_test_helpers");
}
}
const OPENSSL_VERS: &'static str = "1.0.2n";
const OPENSSL_SHA256: &'static str =
"370babb75f278c39e0c50e8c4e7493bc0f18db6867478341a832a982fd15a8fe";
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Openssl {
pub target: Interned<String>,
}
impl Step for Openssl {
type Output = ();
fn should_run(run: ShouldRun) -> ShouldRun {
run.never()
}
fn run(self, builder: &Builder) {
if builder.config.dry_run {
return;
}
let target = self.target;
let out = match builder.openssl_dir(target) {
Some(dir) => dir,
None => return,
};
let stamp = out.join(".stamp");
let mut contents = String::new();
drop(File::open(&stamp).and_then(|mut f| f.read_to_string(&mut contents)));
if contents == OPENSSL_VERS {
return
}
t!(fs::create_dir_all(&out));
let name = format!("openssl-{}.tar.gz", OPENSSL_VERS);
let tarball = out.join(&name);
if !tarball.exists() {
let tmp = tarball.with_extension("tmp");
// originally from https://www.openssl.org/source/...
let url = format!("https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/{}",
name);
let mut last_error = None;
for _ in 0..3 {
let status = Command::new("curl")
.arg("-o").arg(&tmp)
.arg("-f") // make curl fail if the URL does not return HTTP 200
.arg(&url)
.status()
.expect("failed to spawn curl");
// Retry if download failed.
if !status.success() {
last_error = Some(status.to_string());
continue;
}
// Ensure the hash is correct.
let mut shasum = if target.contains("apple") ||
builder.config.build.contains("netbsd") {
let mut cmd = Command::new("shasum");
cmd.arg("-a").arg("256");
cmd
} else {
Command::new("sha256sum")
};
let output = output(&mut shasum.arg(&tmp));
let found = output.split_whitespace().next().unwrap();
// If the hash is wrong, probably the download is incomplete or S3 served an error
// page. In any case, retry.
if found != OPENSSL_SHA256 {
last_error = Some(format!(
"downloaded openssl sha256 different\n\
expected: {}\n\
found: {}\n",
OPENSSL_SHA256,
found
));
continue;
}
// Everything is fine, so exit the retry loop.
last_error = None;
break;
}
if let Some(error) = last_error {
panic!("failed to download openssl source: {}", error);
}
t!(fs::rename(&tmp, &tarball));
}
let obj = out.join(format!("openssl-{}", OPENSSL_VERS));
let dst = builder.openssl_install_dir(target).unwrap();
drop(fs::remove_dir_all(&obj));
drop(fs::remove_dir_all(&dst));
builder.run(Command::new("tar").arg("zxf").arg(&tarball).current_dir(&out));
let mut configure = Command::new("perl");
configure.arg(obj.join("Configure"));
configure.arg(format!("--prefix={}", dst.display()));
configure.arg("no-dso");
configure.arg("no-ssl2");
configure.arg("no-ssl3");
let os = match &*target {
"aarch64-linux-android" => "linux-aarch64",
"aarch64-unknown-linux-gnu" => "linux-aarch64",
"aarch64-unknown-linux-musl" => "linux-aarch64",
"aarch64-unknown-netbsd" => "BSD-generic64",
"arm-linux-androideabi" => "android",
"arm-unknown-linux-gnueabi" => "linux-armv4",
"arm-unknown-linux-gnueabihf" => "linux-armv4",
"armv6-unknown-netbsd-eabihf" => "BSD-generic32",
"armv7-linux-androideabi" => "android-armv7",
"armv7-unknown-linux-gnueabihf" => "linux-armv4",
"armv7-unknown-netbsd-eabihf" => "BSD-generic32",
"i586-unknown-linux-gnu" => "linux-elf",
"i586-unknown-linux-musl" => "linux-elf",
"i686-apple-darwin" => "darwin-i386-cc",
"i686-linux-android" => "android-x86",
"i686-unknown-freebsd" => "BSD-x86-elf",
"i686-unknown-linux-gnu" => "linux-elf",
"i686-unknown-linux-musl" => "linux-elf",
"i686-unknown-netbsd" => "BSD-x86-elf",
"mips-unknown-linux-gnu" => "linux-mips32",
"mips64-unknown-linux-gnuabi64" => "linux64-mips64",
"mips64el-unknown-linux-gnuabi64" => "linux64-mips64",
"mipsel-unknown-linux-gnu" => "linux-mips32",
"powerpc-unknown-linux-gnu" => "linux-ppc",
"powerpc-unknown-linux-gnuspe" => "linux-ppc",
"powerpc-unknown-netbsd" => "BSD-generic32",
"powerpc64-unknown-linux-gnu" => "linux-ppc64",
"powerpc64le-unknown-linux-gnu" => "linux-ppc64le",
"powerpc64le-unknown-linux-musl" => "linux-ppc64le",
"s390x-unknown-linux-gnu" => "linux64-s390x",
"sparc-unknown-linux-gnu" => "linux-sparcv9",
"sparc64-unknown-linux-gnu" => "linux64-sparcv9",
"sparc64-unknown-netbsd" => "BSD-sparc64",
"x86_64-apple-darwin" => "darwin64-x86_64-cc",
"x86_64-linux-android" => "linux-x86_64",
"x86_64-unknown-freebsd" => "BSD-x86_64",
"x86_64-unknown-dragonfly" => "BSD-x86_64",
"x86_64-unknown-linux-gnu" => "linux-x86_64",
"x86_64-unknown-linux-gnux32" => "linux-x32",
"x86_64-unknown-linux-musl" => "linux-x86_64",
"x86_64-unknown-netbsd" => "BSD-x86_64",
_ => panic!("don't know how to configure OpenSSL for {}", target),
};
configure.arg(os);
configure.env("CC", builder.cc(target));
for flag in builder.cflags(target, GitRepo::Rustc) {
configure.arg(flag);
}
// There is no specific os target for android aarch64 or x86_64,
// so we need to pass some extra cflags
if target == "aarch64-linux-android" || target == "x86_64-linux-android" {
configure.arg("-mandroid");
configure.arg("-fomit-frame-pointer");
}
if target == "sparc64-unknown-netbsd" {
// Need -m64 to get assembly generated correctly for sparc64.
configure.arg("-m64");
if builder.config.build.contains("netbsd") {
// Disable sparc64 asm on NetBSD builders, it uses
// m4(1)'s -B flag, which NetBSD m4 does not support.
configure.arg("no-asm");
}
}
// Make PIE binaries
// Non-PIE linker support was removed in Lollipop
// https://source.android.com/security/enhancements/enhancements50
if target == "i686-linux-android" {
configure.arg("no-asm");
}
configure.current_dir(&obj);
builder.info(&format!("Configuring openssl for {}", target));
builder.run_quiet(&mut configure);
builder.info(&format!("Building openssl for {}", target));
builder.run_quiet(Command::new("make").arg("-j1").current_dir(&obj));
builder.info(&format!("Installing openssl for {}", target));
builder.run_quiet(Command::new("make").arg("install").arg("-j1").current_dir(&obj));
let mut f = t!(File::create(&stamp));
t!(f.write_all(OPENSSL_VERS.as_bytes()));
}
}

18
src/bootstrap/test.rs
View File

@ -228,7 +228,8 @@ impl Step for Cargo {
self.host,
"test",
"src/tools/cargo",
SourceType::Submodule);
SourceType::Submodule,
&[]);
if !builder.fail_fast {
cargo.arg("--no-fail-fast");
@ -290,7 +291,8 @@ impl Step for Rls {
host,
"test",
"src/tools/rls",
SourceType::Submodule);
SourceType::Submodule,
&[]);
// Copy `src/tools/rls/test_data` to a writable drive.
let test_workspace_path = builder.out.join("rls-test-data");
@ -352,7 +354,8 @@ impl Step for Rustfmt {
host,
"test",
"src/tools/rustfmt",
SourceType::Submodule);
SourceType::Submodule,
&[]);
let dir = testdir(builder, compiler.host);
t!(fs::create_dir_all(&dir));
@ -407,7 +410,8 @@ impl Step for Miri {
host,
"test",
"src/tools/miri",
SourceType::Submodule);
SourceType::Submodule,
&[]);
// miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
@ -466,7 +470,8 @@ impl Step for Clippy {
host,
"test",
"src/tools/clippy",
SourceType::Submodule);
SourceType::Submodule,
&[]);
// clippy tests need to know about the stage sysroot
cargo.env("SYSROOT", builder.sysroot(compiler));
@ -1777,7 +1782,8 @@ impl Step for CrateRustdoc {
target,
test_kind.subcommand(),
"src/tools/rustdoc",
SourceType::InTree);
SourceType::InTree,
&[]);
if test_kind.subcommand() == "test" && !builder.fail_fast {
cargo.arg("--no-fail-fast");
}

27
src/bootstrap/tool.rs
View File

@ -80,8 +80,8 @@ impl Step for ToolBuild {
"build",
path,
self.source_type,
&self.extra_features,
);
cargo.arg("--features").arg(self.extra_features.join(" "));
let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
@ -208,6 +208,7 @@ pub fn prepare_tool_cargo(
command: &'static str,
path: &'static str,
source_type: SourceType,
extra_features: &[String],
) -> Command {
let mut cargo = builder.cargo(compiler, mode, target, command);
let dir = builder.src.join(path);
@ -221,10 +222,16 @@ pub fn prepare_tool_cargo(
cargo.env("RUSTC_EXTERNAL_TOOL", "1");
}
if let Some(dir) = builder.openssl_install_dir(target) {
cargo.env("OPENSSL_STATIC", "1");
cargo.env("OPENSSL_DIR", dir);
cargo.env("LIBZ_SYS_STATIC", "1");
let mut features = extra_features.iter().cloned().collect::<Vec<_>>();
if builder.build.config.cargo_native_static {
if path.ends_with("cargo") ||
path.ends_with("rls") ||
path.ends_with("clippy") ||
path.ends_with("rustfmt")
{
cargo.env("LIBZ_SYS_STATIC", "1");
features.push("rustc-workspace-hack/all-static".to_string());
}
}
// if tools are using lzma we want to force the build script to build its
@ -244,6 +251,9 @@ pub fn prepare_tool_cargo(
if let Some(date) = info.commit_date() {
cargo.env("CFG_COMMIT_DATE", date);
}
if features.len() > 0 {
cargo.arg("--features").arg(&features.join(", "));
}
cargo
}
@ -439,6 +449,7 @@ impl Step for Rustdoc {
"build",
"src/tools/rustdoc",
SourceType::InTree,
&[],
);
// Most tools don't get debuginfo, but rustdoc should.
@ -495,9 +506,6 @@ impl Step for Cargo {
}
fn run(self, builder: &Builder) -> PathBuf {
builder.ensure(native::Openssl {
target: self.target,
});
// Cargo depends on procedural macros, which requires a full host
// compiler to be available, so we need to depend on that.
builder.ensure(compile::Rustc {
@ -597,9 +605,6 @@ tool_extended!((self, builder),
if clippy.is_some() {
self.extra_features.push("clippy".to_owned());
}
builder.ensure(native::Openssl {
target: self.target,
});
// RLS depends on procedural macros, which requires a full host
// compiler to be available, so we need to depend on that.
builder.ensure(compile::Rustc {

5
src/ci/docker/dist-i686-linux/Dockerfile
View File

@ -83,6 +83,11 @@ RUN ./build-git.sh
COPY dist-x86_64-linux/build-headers.sh /tmp/
RUN ./build-headers.sh
# OpenSSL requires a more recent version of perl
# with so we install newer ones here
COPY dist-x86_64-linux/build-perl.sh /tmp/
RUN ./build-perl.sh
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh

5
src/ci/docker/dist-x86_64-linux/Dockerfile
View File

@ -83,6 +83,11 @@ RUN ./build-git.sh
COPY dist-x86_64-linux/build-headers.sh /tmp/
RUN ./build-headers.sh
# OpenSSL requires a more recent version of perl
# with so we install newer ones here
COPY dist-x86_64-linux/build-perl.sh /tmp/
RUN ./build-perl.sh
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh

29
src/ci/docker/dist-x86_64-linux/build-perl.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright 2018 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
set -ex
source shared.sh
curl https://www.cpan.org/src/5.0/perl-5.28.0.tar.gz | \
tar xzf -
cd perl-5.28.0
# Gotta do some hackery to tell python about our custom OpenSSL build, but other
# than that fairly normal.
CC=gcc \
CFLAGS='-I /rustroot/include' LDFLAGS='-L /rustroot/lib -L /rustroot/lib64' \
hide_output ./configure.gnu
hide_output make -j10
hide_output make install
cd ..
rm -rf perl-5.28.0

2
src/ci/run.sh
View File

@ -39,7 +39,7 @@ fi
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-openssl-static"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static"
if [ "$DIST_SRC" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"

2
src/tools/cargo

@ -1 +1 @@
Subproject commit 09ce4b519535c77d17b818b6dc0b72c6668642c1
Subproject commit 2d0863f657e6f45159fc7412267eee3e659185e5

2
src/tools/rls

@ -1 +1 @@
Subproject commit 440a9855b73b6bf9b5345cf3a79565566f6ef345
Subproject commit 1c755efed6ee265c762f4d3fec73de8a989637a5

17
src/tools/rustc-workspace-hack/Cargo.toml
View File

@ -39,6 +39,7 @@ features = [
"schannel",
"securitybaseapi",
"shellapi",
"sspi",
"synchapi",
"sysinfoapi",
"timezoneapi",
@ -48,10 +49,14 @@ features = [
"wincrypt",
]
[dependencies.serde_json]
version = "1.0.31"
features = ["raw_value"]
[dependencies]
serde_json = { version = "1.0.31", features = ["raw_value"] }
rand = { version = "0.5.5", features = ["i128_support"] }
curl-sys = { version = "0.4.13", optional = true }
[dependencies.rand]
version = "0.5.5"
features = ["i128_support"]
[target.'cfg(not(windows))'.dependencies]
openssl = { version = "0.10.12", optional = true }
[features]
all-static = ['openssl/vendored', 'curl-sys/static-curl']

2
src/tools/rustfmt

@ -1 +1 @@
Subproject commit 90692a5917ac595a9d6b68b88c5f1ad4485670f9
Subproject commit 750b25261380b776de2518fd6863fe63f98d2722