From 640d66c77776cdcac9e959cea9052c2dd51cde98 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 9 Nov 2018 18:38:30 +0100 Subject: [PATCH] Call linker from cg_clif --- Cargo.lock | 79 ++++ Cargo.toml | 3 + build.sh | 16 +- src/archive.rs | 99 +++++ src/lib.rs | 175 +++++++- src/link.rs | 1141 +++++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 1489 insertions(+), 24 deletions(-) create mode 100644 src/archive.rs diff --git a/Cargo.lock b/Cargo.lock index a5ca5e81b20..bb14d457f37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,14 @@ dependencies = [ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cranelift" version = "0.23.0" @@ -258,6 +266,20 @@ dependencies = [ "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "gcc" version = "0.3.55" @@ -361,6 +383,31 @@ dependencies = [ "proc-macro2 0.4.21 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "raw-cpuid" version = "6.0.0" @@ -415,6 +462,14 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.9" @@ -427,12 +482,15 @@ dependencies = [ "ar 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", "cranelift 0.23.0 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-faerie 0.23.0 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-module 0.23.0 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-simplejit 0.23.0 (git+https://github.com/CraneStation/cranelift.git)", "faerie 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", "target-lexicon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -559,6 +617,19 @@ dependencies = [ "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "termcolor" version = "1.0.4" @@ -676,6 +747,7 @@ dependencies = [ "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" "checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cranelift 0.23.0 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum cranelift-bforest 0.23.0 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum cranelift-codegen 0.23.0 (git+https://github.com/CraneStation/cranelift.git)" = "" @@ -692,6 +764,8 @@ dependencies = [ "checksum faerie 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f869e12bcad2b4ff6ac60e01c2354957312fbf0c10dc938f340d58932b3db52" "checksum failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6dd377bcc1b1b7ce911967e3ec24fa19c3224394ec05b54aa7b083d498341ac7" "checksum failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "64c2d913fe8ed3b6c6518eedf4538255b989945c14c2a7d5cbff62a5e2120596" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum gcc 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)" = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" "checksum goblin 0.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "c65cd533b33e3d04c6e393225fa8919ddfcf5862ca8919c7f9a167c312ef41c2" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" @@ -707,12 +781,16 @@ dependencies = [ "checksum proc-macro2 0.4.21 (registry+https://github.com/rust-lang/crates.io-index)" = "ab2fc21ba78ac73e4ff6b3818ece00be4e175ffbef4d0a717d978b48b24150c4" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "53fa22a1994bd0f9372d7a816207d8a2677ad0325b073f5c5332760f0fb62b5c" +"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" +"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" +"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" "checksum raw-cpuid 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41219962ecab392f1e68db9e7ebd972800d4045a128cc23462b384e8c312cde1" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ee84f70c8c08744ea9641a731c7fadb475bf2ecc52d7f627feb833e0b3990467" "checksum regex-syntax 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fbc557aac2b708fe84121caf261346cc2eed71978024337e42eb46b8a252ac6e" "checksum region 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cff177f64744f6f2a2054d231279383e59b0857b4d58dc2e8881e60b58aec8e5" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" @@ -729,6 +807,7 @@ dependencies = [ "checksum syn 0.15.18 (registry+https://github.com/rust-lang/crates.io-index)" = "90c39a061e2f412a9f869540471ab679e85e50c6b05604daf28bc3060f75c430" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum target-lexicon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4af5e2227f0b887d591d3724b796a96eff04226104d872f5b3883fcd427d64b9" +"checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" diff --git a/Cargo.toml b/Cargo.toml index 83e1ac42bdf..cc71244fc4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ faerie = "0.6.0" ar = "0.6.0" bitflags = "1.0.3" byteorder = "1.2.6" +cc = "1.0.25" +libc = "0.2.43" +tempfile = "3.0.4" # Uncomment to use local checkout of cranelift #[patch."https://github.com/CraneStation/cranelift.git"] diff --git a/build.sh b/build.sh index acb9bbc92d7..2b1b0653932 100755 --- a/build.sh +++ b/build.sh @@ -14,11 +14,7 @@ fi build_example_bin() { $RUSTC $2 --crate-name $1 --crate-type bin - - pushd target/out - gcc $1 libmini_core.rlib -o $1_bin - sh -c ./$1_bin || true - popd + sh -c ./target/out/$1 || true } if [[ "$1" == "--release" ]]; then @@ -54,16 +50,12 @@ rm -r target || true time xargo build popd -$RUSTC --sysroot ~/.xargo/HOST example/alloc_example.rs --crate-type bin # TODO linux linker doesn't accept duplicate definitions -#gcc -Wl,--gc-sections target/out/alloc_example ~/.xargo/HOST/lib/rustlib/*/lib/libcore-*.rlib ~/.xargo/HOST/lib/rustlib/*/lib/liballoc-*.rlib ~/.xargo/HOST/lib/rustlib/*/lib/liballoc_system-*.rlib -o target/out/alloc_example_exe -#hyperfine ./target/out/alloc_example_exe +$RUSTC --sysroot ~/.xargo/HOST example/alloc_example.rs --crate-type bin +./target/out/alloc_example $RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin -gcc -Wl,--gc-sections target/out/mod_bench -lc -o target/out/mod_bench_exe - $RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 --crate-name mod_bench_inline -gcc -Wl,--gc-sections target/out/mod_bench_inline -lc -o target/out/mod_bench_inline_exe rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o target/out/mod_bench_llvm_0 -Cpanic=abort rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o target/out/mod_bench_llvm_1 -Cpanic=abort @@ -71,6 +63,6 @@ rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o target/out/mod_benc rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o target/out/mod_bench_llvm_3 -Cpanic=abort echo echo "[Bench] mod_bench" -hyperfine ./target/out/mod_bench{,_inline}_exe ./target/out/mod_bench_llvm_* +hyperfine ./target/out/mod_bench{,_inline} ./target/out/mod_bench_llvm_* cat target/out/log.txt | sort | uniq -c diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 00000000000..1e57ff0f2c4 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,99 @@ +use std::collections::HashMap; +use std::fs::File; +use std::path::PathBuf; + +use crate::prelude::*; + +pub struct ArchiveConfig<'a> { + pub sess: &'a Session, + pub dst: PathBuf, + pub src: Option, + pub lib_search_paths: Vec, +} + +pub struct ArchiveBuilder<'a> { + cfg: ArchiveConfig<'a>, + src_archive: Option>, + src_entries: HashMap, + builder: ar::Builder, + update_symbols: bool, +} + +impl<'a> ArchiveBuilder<'a> { + pub fn new(cfg: ArchiveConfig<'a>) -> Self { + let (src_archive, src_entries) = if let Some(src) = &cfg.src { + let mut archive = ar::Archive::new(File::open(src).unwrap()); + let mut entries = HashMap::new(); + + let mut i = 0; + while let Some(entry) = archive.next_entry() { + let entry = entry.unwrap(); + entries.insert( + String::from_utf8(entry.header().identifier().to_vec()).unwrap(), + i, + ); + i += 1; + } + + (Some(archive), entries) + } else { + (None, HashMap::new()) + }; + + let builder = ar::Builder::new(File::create(&cfg.dst).unwrap()); + + ArchiveBuilder { + cfg, + src_archive, + src_entries, + builder, + update_symbols: false, + } + } + + pub fn src_files(&self) -> Vec { + self.src_entries.keys().cloned().collect() + } + + pub fn remove_file(&mut self, name: &str) { + assert!( + self.src_entries.remove(name).is_some(), + "Tried to remove file not existing in src archive", + ); + } + + pub fn update_symbols(&mut self) { + self.update_symbols = true; + } + + pub fn build(mut self) { + // Add files from original archive + if let Some(mut src_archive) = self.src_archive { + for (_entry_name, entry_idx) in self.src_entries.into_iter() { + let entry = src_archive.jump_to_entry(entry_idx).unwrap(); + let orig_header = entry.header(); + let mut header = + ar::Header::new(orig_header.identifier().to_vec(), orig_header.size()); + header.set_mtime(orig_header.mtime()); + header.set_uid(orig_header.uid()); + header.set_gid(orig_header.gid()); + header.set_mode(orig_header.mode()); + self.builder.append(&header, entry).unwrap(); + } + } + + // Finalize archive + std::mem::drop(self.builder); + + // Run ranlib to be able to link the archive + let status = std::process::Command::new("ranlib") + .arg(self.cfg.dst) + .status() + .expect("Couldn't run ranlib"); + assert!( + status.success(), + "Ranlib exited with code {:?}", + status.code() + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 78e5eed0adb..fb0d6067bff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,9 @@ extern crate rustc_mir; extern crate rustc_target; #[macro_use] extern crate rustc_data_structures; +extern crate rustc_fs_util; +#[macro_use] +extern crate log; extern crate ar; #[macro_use] @@ -37,11 +40,19 @@ use std::sync::mpsc; use syntax::symbol::Symbol; use rustc::dep_graph::DepGraph; -use rustc::middle::cstore::MetadataLoader; -use rustc::session::{config::OutputFilenames, CompileIncomplete}; +use rustc::middle::cstore::{ + self, CrateSource, LibSource, LinkagePreference, MetadataLoader, NativeLibrary, +}; +use rustc::middle::lang_items::LangItem; +use rustc::middle::weak_lang_items; +use rustc::session::{ + config::{self, OutputFilenames, OutputType}, + CompileIncomplete, +}; use rustc::ty::query::Providers; use rustc_codegen_utils::codegen_backend::CodegenBackend; use rustc_codegen_utils::link::out_filename; +use rustc_codegen_utils::linker::LinkerInfo; use cranelift::codegen::settings; use cranelift_faerie::*; @@ -60,6 +71,7 @@ macro_rules! unimpl { mod abi; mod allocator; mod analyze; +mod archive; mod base; mod common; mod constant; @@ -88,7 +100,7 @@ mod prelude { self, subst::Substs, FnSig, Instance, InstanceDef, ParamEnv, PolyFnSig, Ty, TyCtxt, TypeAndMut, TypeFoldable, }; - pub use rustc_codegen_utils::CompiledModule; + pub use rustc_codegen_utils::{CompiledModule, ModuleKind}; pub use rustc_data_structures::{ fx::{FxHashMap, FxHashSet}, indexed_vec::Idx, @@ -108,7 +120,7 @@ mod prelude { pub use crate::abi::*; pub use crate::base::{trans_operand, trans_place}; pub use crate::common::*; - pub use crate::{Caches, CodegenResults}; + pub use crate::{Caches, CodegenResults, CrateInfo}; } pub struct Caches<'tcx> { @@ -127,10 +139,132 @@ impl<'tcx> Caches<'tcx> { struct CraneliftCodegenBackend; +pub struct CrateInfo { + panic_runtime: Option, + compiler_builtins: Option, + profiler_runtime: Option, + sanitizer_runtime: Option, + is_no_builtins: FxHashSet, + native_libraries: FxHashMap>>, + crate_name: FxHashMap, + used_libraries: Lrc>, + link_args: Lrc>, + used_crate_source: FxHashMap>, + used_crates_static: Vec<(CrateNum, LibSource)>, + used_crates_dynamic: Vec<(CrateNum, LibSource)>, + wasm_imports: FxHashMap, + lang_item_to_crate: FxHashMap, + missing_lang_items: FxHashMap>, +} + +impl CrateInfo { + pub fn new(tcx: TyCtxt) -> CrateInfo { + let mut info = CrateInfo { + panic_runtime: None, + compiler_builtins: None, + profiler_runtime: None, + sanitizer_runtime: None, + is_no_builtins: Default::default(), + native_libraries: Default::default(), + used_libraries: tcx.native_libraries(LOCAL_CRATE), + link_args: tcx.link_args(LOCAL_CRATE), + crate_name: Default::default(), + used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic), + used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic), + used_crate_source: Default::default(), + wasm_imports: Default::default(), + lang_item_to_crate: Default::default(), + missing_lang_items: Default::default(), + }; + let lang_items = tcx.lang_items(); + + let load_wasm_items = tcx + .sess + .crate_types + .borrow() + .iter() + .any(|c| *c != config::CrateType::Rlib) + && tcx.sess.opts.target_triple.triple() == "wasm32-unknown-unknown"; + + if load_wasm_items { + info.load_wasm_imports(tcx, LOCAL_CRATE); + } + + let crates = tcx.crates(); + + let n_crates = crates.len(); + info.native_libraries.reserve(n_crates); + info.crate_name.reserve(n_crates); + info.used_crate_source.reserve(n_crates); + info.missing_lang_items.reserve(n_crates); + + for &cnum in crates.iter() { + info.native_libraries + .insert(cnum, tcx.native_libraries(cnum)); + info.crate_name + .insert(cnum, tcx.crate_name(cnum).to_string()); + info.used_crate_source + .insert(cnum, tcx.used_crate_source(cnum)); + if tcx.is_panic_runtime(cnum) { + info.panic_runtime = Some(cnum); + } + if tcx.is_compiler_builtins(cnum) { + info.compiler_builtins = Some(cnum); + } + if tcx.is_profiler_runtime(cnum) { + info.profiler_runtime = Some(cnum); + } + if tcx.is_sanitizer_runtime(cnum) { + info.sanitizer_runtime = Some(cnum); + } + if tcx.is_no_builtins(cnum) { + info.is_no_builtins.insert(cnum); + } + if load_wasm_items { + info.load_wasm_imports(tcx, cnum); + } + let missing = tcx.missing_lang_items(cnum); + for &item in missing.iter() { + if let Ok(id) = lang_items.require(item) { + info.lang_item_to_crate.insert(item, id.krate); + } + } + + // No need to look for lang items that are whitelisted and don't + // actually need to exist. + let missing = missing + .iter() + .cloned() + .filter(|&l| !weak_lang_items::whitelisted(tcx, l)) + .collect(); + info.missing_lang_items.insert(cnum, missing); + } + + return info; + } + + fn load_wasm_imports(&mut self, tcx: TyCtxt, cnum: CrateNum) { + self.wasm_imports.extend( + tcx.wasm_import_module_map(cnum) + .iter() + .map(|(&id, module)| { + let instance = Instance::mono(tcx, id); + let import_name = tcx.symbol_name(instance); + + (import_name.to_string(), module.clone()) + }), + ); + } +} + pub struct CodegenResults { artifact: faerie::Artifact, + modules: Vec, + allocator_module: Option, metadata: Vec, crate_name: Symbol, + crate_info: CrateInfo, + linker_info: LinkerInfo, } impl CodegenBackend for CraneliftCodegenBackend { @@ -261,10 +395,39 @@ impl CodegenBackend for CraneliftCodegenBackend { tcx.sess.abort_if_errors(); + let artifact = faerie_module.finish().artifact; + + let tmp_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, None); + let obj = artifact.emit().unwrap(); + std::fs::write(&tmp_file, obj).unwrap(); + + /*use rustc_mir::monomorphize::partitioning::CodegenUnitExt; + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let ((stats, module), _) = tcx.dep_graph.with_task( + dep_node, + tcx, + cgu_name, + module_codegen, + );*/ + return Box::new(CodegenResults { - artifact: faerie_module.finish().artifact, + artifact, metadata: metadata.raw_data, crate_name: tcx.crate_name(LOCAL_CRATE), + crate_info: CrateInfo::new(tcx), + linker_info: LinkerInfo::new(tcx), + modules: vec![CompiledModule { + name: "dummy".to_string(), + kind: ModuleKind::Regular, + object: Some(tmp_file), + bytecode: None, + bytecode_compressed: None, + }], + //modules: vec![], + allocator_module: None, }); } } @@ -284,7 +447,7 @@ impl CodegenBackend for CraneliftCodegenBackend { let output_name = out_filename(sess, crate_type, &outputs, &res.crate_name.as_str()); match crate_type { CrateType::Rlib => link::link_rlib(sess, &res, output_name), - CrateType::Executable => link::link_bin(sess, &res, output_name), + CrateType::Executable => link::link_bin(sess, &res, &output_name), _ => sess.fatal(&format!("Unsupported crate type: {:?}", crate_type)), } } diff --git a/src/link.rs b/src/link.rs index 14202a4a6d4..f51b9787257 100644 --- a/src/link.rs +++ b/src/link.rs @@ -1,9 +1,36 @@ -use std::fs::File; -use std::path::PathBuf; +use std::env; +use std::fmt; +use std::fs::{self, File}; +use std::io; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; +use cc::windows_registry; +use tempfile::Builder as TempFileBuilder; + +use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind}; +use rustc::middle::dependency_format::Linkage; +use rustc::session::config::{self, CrateType, DebugInfo, OutputType, RUST_CGU_EXT}; +use rustc::session::search_paths::PathKind; use rustc::session::Session; +use rustc::util::common::time; +use rustc_codegen_utils::command::Command; +use rustc_codegen_utils::linker::*; +use rustc_data_structures::fx::FxHashSet; +use rustc_fs_util::fix_windows_verbatim_for_gcc; +use rustc_target::spec::{LinkerFlavor, PanicStrategy, RelroLevel}; +use syntax::attr; -pub(crate) fn link_rlib(sess: &Session, res: &crate::CodegenResults, output_name: PathBuf) { +use crate::prelude::*; + +use crate::archive::{ArchiveBuilder, ArchiveConfig}; +use crate::metadata::METADATA_FILENAME; + +// cg_clif doesn't have bytecode, so this is just a dummy +const RLIB_BYTECODE_EXTENSION: &str = ".cg_clif_bytecode_dummy"; + +pub(crate) fn link_rlib(sess: &Session, res: &CodegenResults, output_name: PathBuf) { let file = File::create(&output_name).unwrap(); let mut builder = ar::Builder::new(file); @@ -42,10 +69,201 @@ pub(crate) fn link_rlib(sess: &Session, res: &crate::CodegenResults, output_name } } -pub(crate) fn link_bin(sess: &Session, res: &crate::CodegenResults, output_name: PathBuf) { +pub(crate) fn link_bin(sess: &Session, codegen_results: &CodegenResults, out_filename: &Path) { + let tmpdir = match TempFileBuilder::new().prefix("rustc").tempdir() { + Ok(tmpdir) => tmpdir, + Err(err) => sess.fatal(&format!("couldn't create a temp dir: {}", err)), + }; + // TODO: link executable - let obj = res.artifact.emit().unwrap(); - std::fs::write(output_name, obj).unwrap(); + let obj = codegen_results.artifact.emit().unwrap(); + std::fs::write(tmpdir.path().join("out".to_string() + RUST_CGU_EXT), obj).unwrap(); + + let (linker, flavor) = linker_and_flavor(sess); + let (pname, mut cmd) = get_linker(sess, &linker, flavor); + + let root = sess.target_filesearch(PathKind::Native).get_lib_path(); + if let Some(args) = sess.target.target.options.pre_link_args.get(&flavor) { + cmd.args(args); + } + if let Some(args) = sess.target.target.options.pre_link_args_crt.get(&flavor) { + if sess.crt_static() { + cmd.args(args); + } + } + if let Some(ref args) = sess.opts.debugging_opts.pre_link_args { + cmd.args(args); + } + cmd.args(&sess.opts.debugging_opts.pre_link_arg); + + for obj in &sess.target.target.options.pre_link_objects_exe { + cmd.arg(root.join(obj)); + } + + if sess.crt_static() { + for obj in &sess.target.target.options.pre_link_objects_exe_crt { + cmd.arg(root.join(obj)); + } + } + + if sess.target.target.options.is_like_emscripten { + cmd.arg("-s"); + cmd.arg(if sess.panic_strategy() == PanicStrategy::Abort { + "DISABLE_EXCEPTION_CATCHING=1" + } else { + "DISABLE_EXCEPTION_CATCHING=0" + }); + } + + { + let target_cpu = "x86_64-apple-darwin"; //::llvm_util::target_cpu(sess); + let mut linker = codegen_results.linker_info.to_linker(cmd, &sess, flavor, target_cpu); + link_args(&mut *linker, flavor, sess, CrateType::Executable, tmpdir.path(), + out_filename, codegen_results); + cmd = linker.finalize(); + } + if let Some(args) = sess.target.target.options.late_link_args.get(&flavor) { + cmd.args(args); + } + for obj in &sess.target.target.options.post_link_objects { + cmd.arg(root.join(obj)); + } + if sess.crt_static() { + for obj in &sess.target.target.options.post_link_objects_crt { + cmd.arg(root.join(obj)); + } + } + if let Some(args) = sess.target.target.options.post_link_args.get(&flavor) { + cmd.args(args); + } + for &(ref k, ref v) in &sess.target.target.options.link_env { + cmd.env(k, v); + } + + if sess.opts.debugging_opts.print_link_args { + println!("{:?}", &cmd); + } + + // May have not found libraries in the right formats. + sess.abort_if_errors(); + + // Invoke the system linker + // + // Note that there's a terribly awful hack that really shouldn't be present + // in any compiler. Here an environment variable is supported to + // automatically retry the linker invocation if the linker looks like it + // segfaulted. + // + // Gee that seems odd, normally segfaults are things we want to know about! + // Unfortunately though in rust-lang/rust#38878 we're experiencing the + // linker segfaulting on Travis quite a bit which is causing quite a bit of + // pain to land PRs when they spuriously fail due to a segfault. + // + // The issue #38878 has some more debugging information on it as well, but + // this unfortunately looks like it's just a race condition in macOS's linker + // with some thread pool working in the background. It seems that no one + // currently knows a fix for this so in the meantime we're left with this... + let retry_on_segfault = env::var("RUSTC_RETRY_LINKER_ON_SEGFAULT").is_ok(); + let mut prog; + let mut i = 0; + loop { + i += 1; + prog = exec_linker(sess, &mut cmd, out_filename, tmpdir.path()); + let output = match prog { + Ok(ref output) => output, + Err(_) => break, + }; + if output.status.success() { + break + } + let mut out = output.stderr.clone(); + out.extend(&output.stdout); + let out = String::from_utf8_lossy(&out); + + // Check to see if the link failed with "unrecognized command line option: + // '-no-pie'" for gcc or "unknown argument: '-no-pie'" for clang. If so, + // reperform the link step without the -no-pie option. This is safe because + // if the linker doesn't support -no-pie then it should not default to + // linking executables as pie. Different versions of gcc seem to use + // different quotes in the error message so don't check for them. + if sess.target.target.options.linker_is_gnu && + flavor != LinkerFlavor::Ld && + (out.contains("unrecognized command line option") || + out.contains("unknown argument")) && + out.contains("-no-pie") && + cmd.get_args().iter().any(|e| e.to_string_lossy() == "-no-pie") { + for arg in cmd.take_args() { + if arg.to_string_lossy() != "-no-pie" { + cmd.arg(arg); + } + } + continue; + } + if !retry_on_segfault || i > 3 { + break + } + let msg_segv = "clang: error: unable to execute command: Segmentation fault: 11"; + let msg_bus = "clang: error: unable to execute command: Bus error: 10"; + if !(out.contains(msg_segv) || out.contains(msg_bus)) { + break + } + } + + match prog { + Ok(prog) => { + if !prog.status.success() { + let mut output = prog.stderr.clone(); + output.extend_from_slice(&prog.stdout); + sess.struct_err(&format!("linking with `{}` failed: {}", + pname.display(), + prog.status)) + .note(&format!("{:?}", &cmd)) + .note(&String::from_utf8_lossy(&output)) + .emit(); + sess.abort_if_errors(); + } + }, + Err(e) => { + let linker_not_found = e.kind() == io::ErrorKind::NotFound; + + let mut linker_error = { + if linker_not_found { + sess.struct_err(&format!("linker `{}` not found", pname.display())) + } else { + sess.struct_err(&format!("could not exec the linker `{}`", pname.display())) + } + }; + + linker_error.note(&e.to_string()); + + if !linker_not_found { + linker_error.note(&format!("{:?}", &cmd)); + } + + linker_error.emit(); + + if sess.target.target.options.is_like_msvc && linker_not_found { + sess.note_without_error("the msvc targets depend on the msvc linker \ + but `link.exe` was not found"); + sess.note_without_error("please ensure that VS 2013, VS 2015 or VS 2017 \ + was installed with the Visual C++ option"); + } + sess.abort_if_errors(); + } + } + + + // On macOS, debuggers need this utility to get run to do some munging of + // the symbols. Note, though, that if the object files are being preserved + // for their debug information there's no need for us to run dsymutil. + if sess.target.target.options.is_like_osx && + sess.opts.debuginfo != DebugInfo::None + { + match Command::new("dsymutil").arg(out_filename).output() { + Ok(..) => {} + Err(e) => sess.fatal(&format!("failed to run dsymutil: {}", e)), + } + } } /* @@ -60,3 +278,914 @@ res.artifact ) .unwrap(); */ + + +fn link_args(cmd: &mut dyn Linker, + flavor: LinkerFlavor, + sess: &Session, + crate_type: config::CrateType, + tmpdir: &Path, + out_filename: &Path, + codegen_results: &CodegenResults) { + + // Linker plugins should be specified early in the list of arguments + cmd.cross_lang_lto(); + + // The default library location, we need this to find the runtime. + // The location of crates will be determined as needed. + let lib_path = sess.target_filesearch(PathKind::All).get_lib_path(); + + // target descriptor + let t = &sess.target.target; + + cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path)); + for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { + cmd.add_object(obj); + } + cmd.output_filename(out_filename); + + // If we're building a dynamic library then some platforms need to make sure + // that all symbols are exported correctly from the dynamic library. + if crate_type != config::CrateType::Executable || + sess.target.target.options.is_like_emscripten { + cmd.export_symbols(tmpdir, crate_type); + } + + let obj = codegen_results.allocator_module + .as_ref() + .and_then(|m| m.object.as_ref()); + if let Some(obj) = obj { + cmd.add_object(obj); + } + + // Try to strip as much out of the generated object by removing unused + // sections if possible. See more comments in linker.rs + if !sess.opts.cg.link_dead_code { + let keep_metadata = crate_type == config::CrateType::Dylib; + cmd.gc_sections(keep_metadata); + } + + let used_link_args = &codegen_results.crate_info.link_args; + + if crate_type == config::CrateType::Executable { + let mut position_independent_executable = false; + + if t.options.position_independent_executables { + let empty_vec = Vec::new(); + let args = sess.opts.cg.link_args.as_ref().unwrap_or(&empty_vec); + let more_args = &sess.opts.cg.link_arg; + let mut args = args.iter().chain(more_args.iter()).chain(used_link_args.iter()); + + if !sess.crt_static() && !args.any(|x| *x == "-static") { + position_independent_executable = true; + } + } + + if position_independent_executable { + cmd.position_independent_executable(); + } else { + // recent versions of gcc can be configured to generate position + // independent executables by default. We have to pass -no-pie to + // explicitly turn that off. Not applicable to ld. + if sess.target.target.options.linker_is_gnu + && flavor != LinkerFlavor::Ld { + cmd.no_position_independent_executable(); + } + } + } + + let relro_level = match sess.opts.debugging_opts.relro_level { + Some(level) => level, + None => t.options.relro_level, + }; + match relro_level { + RelroLevel::Full => { + cmd.full_relro(); + }, + RelroLevel::Partial => { + cmd.partial_relro(); + }, + RelroLevel::Off => { + cmd.no_relro(); + }, + RelroLevel::None => { + }, + } + + // Pass optimization flags down to the linker. + cmd.optimize(); + + // Pass debuginfo flags down to the linker. + cmd.debuginfo(); + + // We want to, by default, prevent the compiler from accidentally leaking in + // any system libraries, so we may explicitly ask linkers to not link to any + // libraries by default. Note that this does not happen for windows because + // windows pulls in some large number of libraries and I couldn't quite + // figure out which subset we wanted. + // + // This is all naturally configurable via the standard methods as well. + if !sess.opts.cg.default_linker_libraries.unwrap_or(false) && + t.options.no_default_libraries + { + cmd.no_default_libraries(); + } + + // Take careful note of the ordering of the arguments we pass to the linker + // here. Linkers will assume that things on the left depend on things to the + // right. Things on the right cannot depend on things on the left. This is + // all formally implemented in terms of resolving symbols (libs on the right + // resolve unknown symbols of libs on the left, but not vice versa). + // + // For this reason, we have organized the arguments we pass to the linker as + // such: + // + // 1. The local object that LLVM just generated + // 2. Local native libraries + // 3. Upstream rust libraries + // 4. Upstream native libraries + // + // The rationale behind this ordering is that those items lower down in the + // list can't depend on items higher up in the list. For example nothing can + // depend on what we just generated (e.g. that'd be a circular dependency). + // Upstream rust libraries are not allowed to depend on our local native + // libraries as that would violate the structure of the DAG, in that + // scenario they are required to link to them as well in a shared fashion. + // + // Note that upstream rust libraries may contain native dependencies as + // well, but they also can't depend on what we just started to add to the + // link line. And finally upstream native libraries can't depend on anything + // in this DAG so far because they're only dylibs and dylibs can only depend + // on other dylibs (e.g. other native deps). + add_local_native_libraries(cmd, sess, codegen_results); + add_upstream_rust_crates(cmd, sess, codegen_results, crate_type, tmpdir); + add_upstream_native_libraries(cmd, sess, codegen_results, crate_type); + + // Tell the linker what we're doing. + if crate_type != config::CrateType::Executable { + cmd.build_dylib(out_filename); + } + if crate_type == config::CrateType::Executable && sess.crt_static() { + cmd.build_static_executable(); + } + + if sess.opts.debugging_opts.pgo_gen.is_some() { + cmd.pgo_gen(); + } + + // Finally add all the linker arguments provided on the command line along + // with any #[link_args] attributes found inside the crate + if let Some(ref args) = sess.opts.cg.link_args { + cmd.args(args); + } + cmd.args(&sess.opts.cg.link_arg); + cmd.args(&used_link_args); +} + +// =========================== +// Copied from https://github.com/rust-lang/rust/blob/942864a000efd74b73e36bda5606b2cdb55ecf39/src/librustc_codegen_llvm/back/link.rs +// =========================== + +fn archive_search_paths(sess: &Session) -> Vec { + let mut search = Vec::new(); + sess.target_filesearch(PathKind::Native).for_each_lib_search_path(|path, _| { + search.push(path.to_path_buf()); + }); + return search; +} + +fn archive_config<'a>(sess: &'a Session, + output: &Path, + input: Option<&Path>) -> ArchiveConfig<'a> { + ArchiveConfig { + sess, + dst: output.to_path_buf(), + src: input.map(|p| p.to_path_buf()), + lib_search_paths: archive_search_paths(sess), + } +} + +// The third parameter is for env vars, used on windows to set up the +// path for MSVC to find its DLLs, and gcc to find its bundled +// toolchain +fn get_linker(sess: &Session, linker: &Path, flavor: LinkerFlavor) -> (PathBuf, Command) { + let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe"); + + // If our linker looks like a batch script on Windows then to execute this + // we'll need to spawn `cmd` explicitly. This is primarily done to handle + // emscripten where the linker is `emcc.bat` and needs to be spawned as + // `cmd /c emcc.bat ...`. + // + // This worked historically but is needed manually since #42436 (regression + // was tagged as #42791) and some more info can be found on #44443 for + // emscripten itself. + let mut cmd = match linker.to_str() { + Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker), + _ => match flavor { + LinkerFlavor::Lld(f) => Command::lld(linker, f), + LinkerFlavor::Msvc + if sess.opts.cg.linker.is_none() && sess.target.target.options.linker.is_none() => + { + Command::new(msvc_tool.as_ref().map(|t| t.path()).unwrap_or(linker)) + }, + _ => Command::new(linker), + } + }; + + // The compiler's sysroot often has some bundled tools, so add it to the + // PATH for the child. + let mut new_path = sess.host_filesearch(PathKind::All) + .get_tools_search_paths(); + let mut msvc_changed_path = false; + if sess.target.target.options.is_like_msvc { + if let Some(ref tool) = msvc_tool { + cmd.args(tool.args()); + for &(ref k, ref v) in tool.env() { + if k == "PATH" { + new_path.extend(env::split_paths(v)); + msvc_changed_path = true; + } else { + cmd.env(k, v); + } + } + } + } + + if !msvc_changed_path { + if let Some(path) = env::var_os("PATH") { + new_path.extend(env::split_paths(&path)); + } + } + cmd.env("PATH", env::join_paths(new_path).unwrap()); + + (linker.to_path_buf(), cmd) +} + +fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { + fn infer_from( + sess: &Session, + linker: Option, + flavor: Option, + ) -> Option<(PathBuf, LinkerFlavor)> { + match (linker, flavor) { + (Some(linker), Some(flavor)) => Some((linker, flavor)), + // only the linker flavor is known; use the default linker for the selected flavor + (None, Some(flavor)) => Some((PathBuf::from(match flavor { + LinkerFlavor::Em => if cfg!(windows) { "emcc.bat" } else { "emcc" }, + LinkerFlavor::Gcc => "cc", + LinkerFlavor::Ld => "ld", + LinkerFlavor::Msvc => "link.exe", + LinkerFlavor::Lld(_) => "lld", + }), flavor)), + (Some(linker), None) => { + let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { + sess.fatal("couldn't extract file stem from specified linker"); + }).to_owned(); + + let flavor = if stem == "emcc" { + LinkerFlavor::Em + } else if stem == "gcc" || stem.ends_with("-gcc") { + LinkerFlavor::Gcc + } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { + LinkerFlavor::Ld + } else if stem == "link" || stem == "lld-link" { + LinkerFlavor::Msvc + } else if stem == "lld" || stem == "rust-lld" { + LinkerFlavor::Lld(sess.target.target.options.lld_flavor) + } else { + // fall back to the value in the target spec + sess.target.target.linker_flavor + }; + + Some((linker, flavor)) + }, + (None, None) => None, + } + } + + // linker and linker flavor specified via command line have precedence over what the target + // specification specifies + if let Some(ret) = infer_from( + sess, + sess.opts.cg.linker.clone(), + sess.opts.debugging_opts.linker_flavor, + ) { + return ret; + } + + if let Some(ret) = infer_from( + sess, + sess.target.target.options.linker.clone().map(PathBuf::from), + Some(sess.target.target.linker_flavor), + ) { + return ret; + } + + bug!("Not enough information provided to determine how to invoke the linker"); +} + +fn exec_linker(sess: &Session, cmd: &mut Command, out_filename: &Path, tmpdir: &Path) + -> io::Result +{ + // When attempting to spawn the linker we run a risk of blowing out the + // size limits for spawning a new process with respect to the arguments + // we pass on the command line. + // + // Here we attempt to handle errors from the OS saying "your list of + // arguments is too big" by reinvoking the linker again with an `@`-file + // that contains all the arguments. The theory is that this is then + // accepted on all linkers and the linker will read all its options out of + // there instead of looking at the command line. + if !cmd.very_likely_to_exceed_some_spawn_limit() { + match cmd.command().stdout(Stdio::piped()).stderr(Stdio::piped()).spawn() { + Ok(child) => { + let output = child.wait_with_output(); + flush_linked_file(&output, out_filename)?; + return output; + } + Err(ref e) if command_line_too_big(e) => { + info!("command line to linker was too big: {}", e); + } + Err(e) => return Err(e) + } + } + + info!("falling back to passing arguments to linker via an @-file"); + let mut cmd2 = cmd.clone(); + let mut args = String::new(); + for arg in cmd2.take_args() { + args.push_str(&Escape { + arg: arg.to_str().unwrap(), + is_like_msvc: sess.target.target.options.is_like_msvc, + }.to_string()); + args.push_str("\n"); + } + let file = tmpdir.join("linker-arguments"); + let bytes = if sess.target.target.options.is_like_msvc { + let mut out = Vec::with_capacity((1 + args.len()) * 2); + // start the stream with a UTF-16 BOM + for c in iter::once(0xFEFF).chain(args.encode_utf16()) { + // encode in little endian + out.push(c as u8); + out.push((c >> 8) as u8); + } + out + } else { + args.into_bytes() + }; + fs::write(&file, &bytes)?; + cmd2.arg(format!("@{}", file.display())); + info!("invoking linker {:?}", cmd2); + let output = cmd2.output(); + flush_linked_file(&output, out_filename)?; + return output; + + #[cfg(unix)] + fn flush_linked_file(_: &io::Result, _: &Path) -> io::Result<()> { + Ok(()) + } + + #[cfg(windows)] + fn flush_linked_file(command_output: &io::Result, out_filename: &Path) + -> io::Result<()> + { + // On Windows, under high I/O load, output buffers are sometimes not flushed, + // even long after process exit, causing nasty, non-reproducible output bugs. + // + // File::sync_all() calls FlushFileBuffers() down the line, which solves the problem. + // + // А full writeup of the original Chrome bug can be found at + // randomascii.wordpress.com/2018/02/25/compiler-bug-linker-bug-windows-kernel-bug/amp + + if let &Ok(ref out) = command_output { + if out.status.success() { + if let Ok(of) = fs::OpenOptions::new().write(true).open(out_filename) { + of.sync_all()?; + } + } + } + + Ok(()) + } + + #[cfg(unix)] + fn command_line_too_big(err: &io::Error) -> bool { + err.raw_os_error() == Some(::libc::E2BIG) + } + + #[cfg(windows)] + fn command_line_too_big(err: &io::Error) -> bool { + const ERROR_FILENAME_EXCED_RANGE: i32 = 206; + err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE) + } + + struct Escape<'a> { + arg: &'a str, + is_like_msvc: bool, + } + + impl<'a> fmt::Display for Escape<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.is_like_msvc { + // This is "documented" at + // https://msdn.microsoft.com/en-us/library/4xdcbak7.aspx + // + // Unfortunately there's not a great specification of the + // syntax I could find online (at least) but some local + // testing showed that this seemed sufficient-ish to catch + // at least a few edge cases. + write!(f, "\"")?; + for c in self.arg.chars() { + match c { + '"' => write!(f, "\\{}", c)?, + c => write!(f, "{}", c)?, + } + } + write!(f, "\"")?; + } else { + // This is documented at https://linux.die.net/man/1/ld, namely: + // + // > Options in file are separated by whitespace. A whitespace + // > character may be included in an option by surrounding the + // > entire option in either single or double quotes. Any + // > character (including a backslash) may be included by + // > prefixing the character to be included with a backslash. + // + // We put an argument on each line, so all we need to do is + // ensure the line is interpreted as one whole argument. + for c in self.arg.chars() { + match c { + '\\' | + ' ' => write!(f, "\\{}", c)?, + c => write!(f, "{}", c)?, + } + } + } + Ok(()) + } + } +} + +// # Rust Crate linking +// +// Rust crates are not considered at all when creating an rlib output. All +// dependencies will be linked when producing the final output (instead of +// the intermediate rlib version) +fn add_upstream_rust_crates(cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + crate_type: config::CrateType, + tmpdir: &Path) { + // All of the heavy lifting has previously been accomplished by the + // dependency_format module of the compiler. This is just crawling the + // output of that module, adding crates as necessary. + // + // Linking to a rlib involves just passing it to the linker (the linker + // will slurp up the object files inside), and linking to a dynamic library + // involves just passing the right -l flag. + + let formats = sess.dependency_formats.borrow(); + let data = formats.get(&crate_type).unwrap(); + + // Invoke get_used_crates to ensure that we get a topological sorting of + // crates. + let deps = &codegen_results.crate_info.used_crates_dynamic; + + // There's a few internal crates in the standard library (aka libcore and + // libstd) which actually have a circular dependence upon one another. This + // currently arises through "weak lang items" where libcore requires things + // like `rust_begin_unwind` but libstd ends up defining it. To get this + // circular dependence to work correctly in all situations we'll need to be + // sure to correctly apply the `--start-group` and `--end-group` options to + // GNU linkers, otherwise if we don't use any other symbol from the standard + // library it'll get discarded and the whole application won't link. + // + // In this loop we're calculating the `group_end`, after which crate to + // pass `--end-group` and `group_start`, before which crate to pass + // `--start-group`. We currently do this by passing `--end-group` after + // the first crate (when iterating backwards) that requires a lang item + // defined somewhere else. Once that's set then when we've defined all the + // necessary lang items we'll pass `--start-group`. + // + // Note that this isn't amazing logic for now but it should do the trick + // for the current implementation of the standard library. + let mut group_end = None; + let mut group_start = None; + let mut end_with = FxHashSet::default(); + let info = &codegen_results.crate_info; + for &(cnum, _) in deps.iter().rev() { + if let Some(missing) = info.missing_lang_items.get(&cnum) { + end_with.extend(missing.iter().cloned()); + if end_with.len() > 0 && group_end.is_none() { + group_end = Some(cnum); + } + } + end_with.retain(|item| info.lang_item_to_crate.get(item) != Some(&cnum)); + if end_with.len() == 0 && group_end.is_some() { + group_start = Some(cnum); + break + } + } + + // If we didn't end up filling in all lang items from upstream crates then + // we'll be filling it in with our crate. This probably means we're the + // standard library itself, so skip this for now. + if group_end.is_some() && group_start.is_none() { + group_end = None; + } + + let mut compiler_builtins = None; + + for &(cnum, _) in deps.iter() { + if group_start == Some(cnum) { + cmd.group_start(); + } + + // We may not pass all crates through to the linker. Some crates may + // appear statically in an existing dylib, meaning we'll pick up all the + // symbols from the dylib. + let src = &codegen_results.crate_info.used_crate_source[&cnum]; + match data[cnum.as_usize() - 1] { + _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => { + add_static_crate(cmd, sess, codegen_results, tmpdir, crate_type, cnum); + } + _ if codegen_results.crate_info.sanitizer_runtime == Some(cnum) => { + link_sanitizer_runtime(cmd, sess, codegen_results, tmpdir, cnum); + } + // compiler-builtins are always placed last to ensure that they're + // linked correctly. + _ if codegen_results.crate_info.compiler_builtins == Some(cnum) => { + assert!(compiler_builtins.is_none()); + compiler_builtins = Some(cnum); + } + Linkage::NotLinked | + Linkage::IncludedFromDylib => {} + Linkage::Static => { + add_static_crate(cmd, sess, codegen_results, tmpdir, crate_type, cnum); + } + Linkage::Dynamic => { + add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0) + } + } + + if group_end == Some(cnum) { + cmd.group_end(); + } + } + + // compiler-builtins are always placed last to ensure that they're + // linked correctly. + // We must always link the `compiler_builtins` crate statically. Even if it + // was already "included" in a dylib (e.g. `libstd` when `-C prefer-dynamic` + // is used) + if let Some(cnum) = compiler_builtins { + add_static_crate(cmd, sess, codegen_results, tmpdir, crate_type, cnum); + } + + // Converts a library file-stem into a cc -l argument + fn unlib<'a>(config: &config::Config, stem: &'a str) -> &'a str { + if stem.starts_with("lib") && !config.target.options.is_like_windows { + &stem[3..] + } else { + stem + } + } + + // We must link the sanitizer runtime using -Wl,--whole-archive but since + // it's packed in a .rlib, it contains stuff that are not objects that will + // make the linker error. So we must remove those bits from the .rlib before + // linking it. + fn link_sanitizer_runtime(cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + tmpdir: &Path, + cnum: CrateNum) { + let src = &codegen_results.crate_info.used_crate_source[&cnum]; + let cratepath = &src.rlib.as_ref().unwrap().0; + + if sess.target.target.options.is_like_osx { + // On Apple platforms, the sanitizer is always built as a dylib, and + // LLVM will link to `@rpath/*.dylib`, so we need to specify an + // rpath to the library as well (the rpath should be absolute, see + // PR #41352 for details). + // + // FIXME: Remove this logic into librustc_*san once Cargo supports it + let rpath = cratepath.parent().unwrap(); + let rpath = rpath.to_str().expect("non-utf8 component in path"); + cmd.args(&["-Wl,-rpath".into(), "-Xlinker".into(), rpath.into()]); + } + + let dst = tmpdir.join(cratepath.file_name().unwrap()); + let cfg = archive_config(sess, &dst, Some(cratepath)); + let mut archive = ArchiveBuilder::new(cfg); + archive.update_symbols(); + + for f in archive.src_files() { + if f.ends_with(RLIB_BYTECODE_EXTENSION) || f == METADATA_FILENAME { + archive.remove_file(&f); + continue + } + } + + archive.build(); + + cmd.link_whole_rlib(&dst); + } + + // Adds the static "rlib" versions of all crates to the command line. + // There's a bit of magic which happens here specifically related to LTO and + // dynamic libraries. Specifically: + // + // * For LTO, we remove upstream object files. + // * For dylibs we remove metadata and bytecode from upstream rlibs + // + // When performing LTO, almost(*) all of the bytecode from the upstream + // libraries has already been included in our object file output. As a + // result we need to remove the object files in the upstream libraries so + // the linker doesn't try to include them twice (or whine about duplicate + // symbols). We must continue to include the rest of the rlib, however, as + // it may contain static native libraries which must be linked in. + // + // (*) Crates marked with `#![no_builtins]` don't participate in LTO and + // their bytecode wasn't included. The object files in those libraries must + // still be passed to the linker. + // + // When making a dynamic library, linkers by default don't include any + // object files in an archive if they're not necessary to resolve the link. + // We basically want to convert the archive (rlib) to a dylib, though, so we + // *do* want everything included in the output, regardless of whether the + // linker thinks it's needed or not. As a result we must use the + // --whole-archive option (or the platform equivalent). When using this + // option the linker will fail if there are non-objects in the archive (such + // as our own metadata and/or bytecode). All in all, for rlibs to be + // entirely included in dylibs, we need to remove all non-object files. + // + // Note, however, that if we're not doing LTO or we're not producing a dylib + // (aka we're making an executable), we can just pass the rlib blindly to + // the linker (fast) because it's fine if it's not actually included as + // we're at the end of the dependency chain. + fn add_static_crate(cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + tmpdir: &Path, + crate_type: config::CrateType, + cnum: CrateNum) { + let src = &codegen_results.crate_info.used_crate_source[&cnum]; + let cratepath = &src.rlib.as_ref().unwrap().0; + + // See the comment above in `link_staticlib` and `link_rlib` for why if + // there's a static library that's not relevant we skip all object + // files. + let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; + let skip_native = native_libs.iter().any(|lib| { + lib.kind == NativeLibraryKind::NativeStatic && !relevant_lib(sess, lib) + }); + + if (!are_upstream_rust_objects_already_included(sess) || + ignored_for_lto(sess, &codegen_results.crate_info, cnum)) && + crate_type != config::CrateType::Dylib && + !skip_native { + cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); + return + } + + let dst = tmpdir.join(cratepath.file_name().unwrap()); + let name = cratepath.file_name().unwrap().to_str().unwrap(); + let name = &name[3..name.len() - 5]; // chop off lib/.rlib + + time(sess, &format!("altering {}.rlib", name), || { + let cfg = archive_config(sess, &dst, Some(cratepath)); + let mut archive = ArchiveBuilder::new(cfg); + archive.update_symbols(); + + let mut any_objects = false; + for f in archive.src_files() { + if f.ends_with(RLIB_BYTECODE_EXTENSION) || f == METADATA_FILENAME { + archive.remove_file(&f); + continue + } + + let canonical = f.replace("-", "_"); + let canonical_name = name.replace("-", "_"); + + // Look for `.rcgu.o` at the end of the filename to conclude + // that this is a Rust-related object file. + fn looks_like_rust(s: &str) -> bool { + let path = Path::new(s); + let ext = path.extension().and_then(|s| s.to_str()); + if ext != Some(OutputType::Object.extension()) { + return false + } + let ext2 = path.file_stem() + .and_then(|s| Path::new(s).extension()) + .and_then(|s| s.to_str()); + ext2 == Some(RUST_CGU_EXT) + } + + let is_rust_object = + canonical.starts_with(&canonical_name) && + looks_like_rust(&f); + + // If we've been requested to skip all native object files + // (those not generated by the rust compiler) then we can skip + // this file. See above for why we may want to do this. + let skip_because_cfg_say_so = skip_native && !is_rust_object; + + // If we're performing LTO and this is a rust-generated object + // file, then we don't need the object file as it's part of the + // LTO module. Note that `#![no_builtins]` is excluded from LTO, + // though, so we let that object file slide. + let skip_because_lto = are_upstream_rust_objects_already_included(sess) && + is_rust_object && + (sess.target.target.options.no_builtins || + !codegen_results.crate_info.is_no_builtins.contains(&cnum)); + + if skip_because_cfg_say_so || skip_because_lto { + archive.remove_file(&f); + } else { + any_objects = true; + } + } + + if !any_objects { + return + } + archive.build(); + + // If we're creating a dylib, then we need to include the + // whole of each object in our archive into that artifact. This is + // because a `dylib` can be reused as an intermediate artifact. + // + // Note, though, that we don't want to include the whole of a + // compiler-builtins crate (e.g. compiler-rt) because it'll get + // repeatedly linked anyway. + if crate_type == config::CrateType::Dylib && + codegen_results.crate_info.compiler_builtins != Some(cnum) { + cmd.link_whole_rlib(&fix_windows_verbatim_for_gcc(&dst)); + } else { + cmd.link_rlib(&fix_windows_verbatim_for_gcc(&dst)); + } + }); + } + + // Same thing as above, but for dynamic crates instead of static crates. + fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { + // If we're performing LTO, then it should have been previously required + // that all upstream rust dependencies were available in an rlib format. + assert!(!are_upstream_rust_objects_already_included(sess)); + + // Just need to tell the linker about where the library lives and + // what its name is + let parent = cratepath.parent(); + if let Some(dir) = parent { + cmd.include_path(&fix_windows_verbatim_for_gcc(dir)); + } + let filestem = cratepath.file_stem().unwrap().to_str().unwrap(); + cmd.link_rust_dylib(&unlib(&sess.target, filestem), + parent.unwrap_or(Path::new(""))); + } +} + +// # Native library linking +// +// User-supplied library search paths (-L on the command line). These are +// the same paths used to find Rust crates, so some of them may have been +// added already by the previous crate linking code. This only allows them +// to be found at compile time so it is still entirely up to outside +// forces to make sure that library can be found at runtime. +// +// Also note that the native libraries linked here are only the ones located +// in the current crate. Upstream crates with native library dependencies +// may have their native library pulled in above. +fn add_local_native_libraries(cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults) { + sess.target_filesearch(PathKind::All).for_each_lib_search_path(|path, k| { + match k { + PathKind::Framework => { cmd.framework_path(path); } + _ => { cmd.include_path(&fix_windows_verbatim_for_gcc(path)); } + } + }); + + let relevant_libs = codegen_results.crate_info.used_libraries.iter().filter(|l| { + relevant_lib(sess, l) + }); + + let search_path = archive_search_paths(sess); + for lib in relevant_libs { + let name = match lib.name { + Some(ref l) => l, + None => continue, + }; + match lib.kind { + NativeLibraryKind::NativeUnknown => cmd.link_dylib(&name.as_str()), + NativeLibraryKind::NativeFramework => cmd.link_framework(&name.as_str()), + NativeLibraryKind::NativeStaticNobundle => cmd.link_staticlib(&name.as_str()), + NativeLibraryKind::NativeStatic => cmd.link_whole_staticlib(&name.as_str(), + &search_path) + } + } +} + +// Link in all of our upstream crates' native dependencies. Remember that +// all of these upstream native dependencies are all non-static +// dependencies. We've got two cases then: +// +// 1. The upstream crate is an rlib. In this case we *must* link in the +// native dependency because the rlib is just an archive. +// +// 2. The upstream crate is a dylib. In order to use the dylib, we have to +// have the dependency present on the system somewhere. Thus, we don't +// gain a whole lot from not linking in the dynamic dependency to this +// crate as well. +// +// The use case for this is a little subtle. In theory the native +// dependencies of a crate are purely an implementation detail of the crate +// itself, but the problem arises with generic and inlined functions. If a +// generic function calls a native function, then the generic function must +// be instantiated in the target crate, meaning that the native symbol must +// also be resolved in the target crate. +fn add_upstream_native_libraries(cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + crate_type: config::CrateType) { + // Be sure to use a topological sorting of crates because there may be + // interdependencies between native libraries. When passing -nodefaultlibs, + // for example, almost all native libraries depend on libc, so we have to + // make sure that's all the way at the right (liblibc is near the base of + // the dependency chain). + // + // This passes RequireStatic, but the actual requirement doesn't matter, + // we're just getting an ordering of crate numbers, we're not worried about + // the paths. + let formats = sess.dependency_formats.borrow(); + let data = formats.get(&crate_type).unwrap(); + + let crates = &codegen_results.crate_info.used_crates_static; + for &(cnum, _) in crates { + for lib in codegen_results.crate_info.native_libraries[&cnum].iter() { + let name = match lib.name { + Some(ref l) => l, + None => continue, + }; + if !relevant_lib(sess, &lib) { + continue + } + match lib.kind { + NativeLibraryKind::NativeUnknown => cmd.link_dylib(&name.as_str()), + NativeLibraryKind::NativeFramework => cmd.link_framework(&name.as_str()), + NativeLibraryKind::NativeStaticNobundle => { + // Link "static-nobundle" native libs only if the crate they originate from + // is being linked statically to the current crate. If it's linked dynamically + // or is an rlib already included via some other dylib crate, the symbols from + // native libs will have already been included in that dylib. + if data[cnum.as_usize() - 1] == Linkage::Static { + cmd.link_staticlib(&name.as_str()) + } + }, + // ignore statically included native libraries here as we've + // already included them when we included the rust library + // previously + NativeLibraryKind::NativeStatic => {} + } + } + } +} + +/// Returns a boolean indicating whether the specified crate should be ignored +/// during LTO. +/// +/// Crates ignored during LTO are not lumped together in the "massive object +/// file" that we create and are linked in their normal rlib states. See +/// comments below for what crates do not participate in LTO. +/// +/// It's unusual for a crate to not participate in LTO. Typically only +/// compiler-specific and unstable crates have a reason to not participate in +/// LTO. +fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool { + // If our target enables builtin function lowering in LLVM then the + // crates providing these functions don't participate in LTO (e.g. + // no_builtins or compiler builtins crates). + !sess.target.target.options.no_builtins && + (info.is_no_builtins.contains(&cnum) || info.compiler_builtins == Some(cnum)) +} + +fn relevant_lib(sess: &Session, lib: &NativeLibrary) -> bool { + match lib.cfg { + Some(ref cfg) => attr::cfg_matches(cfg, &sess.parse_sess, None), + None => true, + } +} + +fn are_upstream_rust_objects_already_included(sess: &Session) -> bool { + match sess.lto() { + Lto::Fat => true, + Lto::Thin => { + // If we defer LTO to the linker, we haven't run LTO ourselves, so + // any upstream object files have not been copied yet. + !sess.opts.debugging_opts.cross_lang_lto.enabled() + } + Lto::No | + Lto::ThinLocal => false, + } +}