From 65e8a13441cb4169ce4fb43be7f1369d5c9d71e0 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 13 May 2016 20:48:32 -0400 Subject: [PATCH] Adapt backend to trans::partitioning dictating the codegen-unit setup. --- src/librustc/session/config.rs | 65 +++++++++-- src/librustc_driver/driver.rs | 2 +- src/librustc_trans/back/link.rs | 15 +-- src/librustc_trans/back/lto.rs | 7 +- src/librustc_trans/back/write.rs | 170 +++++++++++++++++------------ src/librustc_trans/base.rs | 8 +- src/librustc_trans/lib.rs | 3 +- src/librustc_trans/partitioning.rs | 6 +- 8 files changed, 181 insertions(+), 95 deletions(-) diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index a3799006192..5ccc96210be 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -197,23 +197,70 @@ pub struct OutputFilenames { pub outputs: HashMap>, } +/// Codegen unit names generated by the numbered naming scheme will contain this +/// marker right before the index of the codegen unit. +pub const NUMBERED_CODEGEN_UNIT_MARKER: &'static str = ".cgu-"; + impl OutputFilenames { pub fn path(&self, flavor: OutputType) -> PathBuf { self.outputs.get(&flavor).and_then(|p| p.to_owned()) .or_else(|| self.single_output_file.clone()) - .unwrap_or_else(|| self.temp_path(flavor)) + .unwrap_or_else(|| self.temp_path(flavor, None)) } - pub fn temp_path(&self, flavor: OutputType) -> PathBuf { + /// Get the path where a compilation artifact of the given type for the + /// given codegen unit should be placed on disk. If codegen_unit_name is + /// None, a path distinct from those of any codegen unit will be generated. + pub fn temp_path(&self, + flavor: OutputType, + codegen_unit_name: Option<&str>) + -> PathBuf { + let extension = match flavor { + OutputType::Bitcode => "bc", + OutputType::Assembly => "s", + OutputType::LlvmAssembly => "ll", + OutputType::Object => "o", + OutputType::DepInfo => "d", + OutputType::Exe => "", + }; + + self.temp_path_ext(extension, codegen_unit_name) + } + + /// Like temp_path, but also supports things where there is no corresponding + /// OutputType, like no-opt-bitcode or lto-bitcode. + pub fn temp_path_ext(&self, + ext: &str, + codegen_unit_name: Option<&str>) + -> PathBuf { let base = self.out_directory.join(&self.filestem()); - match flavor { - OutputType::Bitcode => base.with_extension("bc"), - OutputType::Assembly => base.with_extension("s"), - OutputType::LlvmAssembly => base.with_extension("ll"), - OutputType::Object => base.with_extension("o"), - OutputType::DepInfo => base.with_extension("d"), - OutputType::Exe => base, + + let mut extension = String::new(); + + if let Some(codegen_unit_name) = codegen_unit_name { + if codegen_unit_name.contains(NUMBERED_CODEGEN_UNIT_MARKER) { + // If we use the numbered naming scheme for modules, we don't want + // the files to look like ... + // but simply .. + let marker_offset = codegen_unit_name.rfind(NUMBERED_CODEGEN_UNIT_MARKER) + .unwrap(); + let index_offset = marker_offset + NUMBERED_CODEGEN_UNIT_MARKER.len(); + extension.push_str(&codegen_unit_name[index_offset .. ]); + } else { + extension.push_str(codegen_unit_name); + }; } + + if !ext.is_empty() { + if !extension.is_empty() { + extension.push_str("."); + } + + extension.push_str(ext); + } + + let path = base.with_extension(&extension[..]); + path } pub fn with_extension(&self, extension: &str) -> PathBuf { diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index 277789f5312..eef2b6e6f37 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -1081,7 +1081,7 @@ pub fn phase_5_run_llvm_passes(sess: &Session, // Remove assembly source, unless --save-temps was specified if !sess.opts.cg.save_temps { - fs::remove_file(&outputs.temp_path(OutputType::Assembly)).unwrap(); + fs::remove_file(&outputs.temp_path(OutputType::Assembly, None)).unwrap(); } } else { time(sess.time_passes(), diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index 744712b22b0..a9f3d2f8a17 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -205,7 +205,7 @@ pub fn link_binary(sess: &Session, // Remove the temporary object file and metadata if we aren't saving temps if !sess.opts.cg.save_temps { - for obj in object_filenames(sess, outputs) { + for obj in object_filenames(trans, outputs) { remove(sess, &obj); } remove(sess, &outputs.with_extension("metadata.o")); @@ -316,7 +316,7 @@ fn link_binary_output(sess: &Session, crate_type: config::CrateType, outputs: &OutputFilenames, crate_name: &str) -> PathBuf { - let objects = object_filenames(sess, outputs); + let objects = object_filenames(trans, outputs); let default_filename = filename_for_input(sess, crate_type, crate_name, outputs); let out_filename = outputs.outputs.get(&OutputType::Exe) @@ -356,10 +356,11 @@ fn link_binary_output(sess: &Session, out_filename } -fn object_filenames(sess: &Session, outputs: &OutputFilenames) -> Vec { - (0..sess.opts.cg.codegen_units).map(|i| { - let ext = format!("{}.o", i); - outputs.temp_path(OutputType::Object).with_extension(&ext) +fn object_filenames(trans: &CrateTranslation, + outputs: &OutputFilenames) + -> Vec { + trans.modules.iter().map(|module| { + outputs.temp_path(OutputType::Object, Some(&module.name[..])) }).collect() } @@ -497,7 +498,7 @@ fn link_rlib<'a>(sess: &'a Session, ab.add_file(&bc_deflated_filename); // See the bottom of back::write::run_passes for an explanation - // of when we do and don't keep .0.bc files around. + // of when we do and don't keep .#module-name#.bc files around. let user_wants_numbered_bitcode = sess.opts.output_types.contains_key(&OutputType::Bitcode) && sess.opts.cg.codegen_units > 1; diff --git a/src/librustc_trans/back/lto.rs b/src/librustc_trans/back/lto.rs index 31bc11fb215..69e4a50804f 100644 --- a/src/librustc_trans/back/lto.rs +++ b/src/librustc_trans/back/lto.rs @@ -22,12 +22,12 @@ use libc; use flate; use std::ffi::CString; +use std::path::Path; pub fn run(sess: &session::Session, llmod: ModuleRef, tm: TargetMachineRef, reachable: &[String], config: &ModuleConfig, - name_extra: &str, - output_names: &config::OutputFilenames) { + temp_no_opt_bc_filename: &Path) { if sess.opts.cg.prefer_dynamic { sess.struct_err("cannot prefer dynamic linking when performing LTO") .note("only 'staticlib', 'bin', and 'cdylib' outputs are \ @@ -132,8 +132,7 @@ pub fn run(sess: &session::Session, llmod: ModuleRef, } if sess.opts.cg.save_temps { - let path = output_names.with_extension(&format!("{}.no-opt.lto.bc", name_extra)); - let cstr = path2cstr(&path); + let cstr = path2cstr(temp_no_opt_bc_filename); unsafe { llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); } diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index ec20381d189..28804e4ed9e 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -423,9 +423,9 @@ unsafe extern "C" fn diagnostic_handler(info: DiagnosticInfoRef, user: *mut c_vo unsafe fn optimize_and_codegen(cgcx: &CodegenContext, mtrans: ModuleTranslation, config: ModuleConfig, - name_extra: String, output_names: OutputFilenames) { - let ModuleTranslation { llmod, llcx } = mtrans; + let llmod = mtrans.llmod; + let llcx = mtrans.llcx; let tm = config.tm; // llcx doesn't outlive this function, so we can put this on the stack. @@ -438,9 +438,10 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, llvm::LLVMSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, fv); llvm::LLVMContextSetDiagnosticHandler(llcx, diagnostic_handler, fv); + let module_name = Some(&mtrans.name[..]); + if config.emit_no_opt_bc { - let ext = format!("{}.no-opt.bc", name_extra); - let out = output_names.with_extension(&ext); + let out = output_names.temp_path_ext("no-opt.bc", module_name); let out = path2cstr(&out); llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); } @@ -512,13 +513,18 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, match cgcx.lto_ctxt { Some((sess, reachable)) if sess.lto() => { - time(sess.time_passes(), "all lto passes", || - lto::run(sess, llmod, tm, reachable, &config, - &name_extra, &output_names)); - + time(sess.time_passes(), "all lto passes", || { + let temp_no_opt_bc_filename = + output_names.temp_path_ext("no-opt.lto.bc", module_name); + lto::run(sess, + llmod, + tm, + reachable, + &config, + &temp_no_opt_bc_filename); + }); if config.emit_lto_bc { - let name = format!("{}.lto.bc", name_extra); - let out = output_names.with_extension(&name); + let out = output_names.temp_path_ext("lto.bc", module_name); let out = path2cstr(&out); llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); } @@ -556,8 +562,8 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, let write_obj = config.emit_obj && !config.obj_is_bitcode; let copy_bc_to_obj = config.emit_obj && config.obj_is_bitcode; - let bc_out = output_names.with_extension(&format!("{}.bc", name_extra)); - let obj_out = output_names.with_extension(&format!("{}.o", name_extra)); + let bc_out = output_names.temp_path(OutputType::Bitcode, module_name); + let obj_out = output_names.temp_path(OutputType::Object, module_name); if write_bc { let bc_out_c = path2cstr(&bc_out); @@ -566,8 +572,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, time(config.time_passes, &format!("codegen passes [{}]", cgcx.worker), || { if config.emit_ir { - let ext = format!("{}.ll", name_extra); - let out = output_names.with_extension(&ext); + let out = output_names.temp_path(OutputType::LlvmAssembly, module_name); let out = path2cstr(&out); with_codegen(tm, llmod, config.no_builtins, |cpm| { llvm::LLVMRustPrintModule(cpm, llmod, out.as_ptr()); @@ -576,7 +581,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, } if config.emit_asm { - let path = output_names.with_extension(&format!("{}.s", name_extra)); + let path = output_names.temp_path(OutputType::Assembly, module_name); // We can't use the same module for asm and binary output, because that triggers // various errors like invalid IR or broken binaries, so we might have to clone the @@ -713,27 +718,29 @@ pub fn run_passes(sess: &Session, { let work = build_work_item(sess, - trans.metadata_module, + trans.metadata_module.clone(), metadata_config.clone(), - crate_output.clone(), - "metadata".to_string()); + crate_output.clone()); work_items.push(work); } - for (index, mtrans) in trans.modules.iter().enumerate() { + for mtrans in trans.modules.iter() { let work = build_work_item(sess, - *mtrans, + mtrans.clone(), modules_config.clone(), - crate_output.clone(), - format!("{}", index)); + crate_output.clone()); work_items.push(work); } // Process the work items, optionally using worker threads. - if sess.opts.cg.codegen_units == 1 { + // NOTE: This code is not really adapted to incremental compilation where + // the compiler decides the number of codegen units (and will + // potentially create hundreds of them). + let num_workers = work_items.len() - 1; + if num_workers == 1 { run_work_singlethreaded(sess, &trans.reachable, work_items); } else { - run_work_multithreaded(sess, work_items, sess.opts.cg.codegen_units); + run_work_multithreaded(sess, work_items, num_workers); } // All codegen is finished. @@ -748,32 +755,42 @@ pub fn run_passes(sess: &Session, } }; - let copy_if_one_unit = |ext: &str, - output_type: OutputType, + let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| { - if sess.opts.cg.codegen_units == 1 { + if trans.modules.len() == 1 { // 1) Only one codegen unit. In this case it's no difficulty // to copy `foo.0.x` to `foo.x`. - copy_gracefully(&crate_output.with_extension(ext), + let module_name = Some(&(trans.modules[0].name)[..]); + let path = crate_output.temp_path(output_type, module_name); + copy_gracefully(&path, &crate_output.path(output_type)); if !sess.opts.cg.save_temps && !keep_numbered { - // The user just wants `foo.x`, not `foo.0.x`. - remove(sess, &crate_output.with_extension(ext)); + // The user just wants `foo.x`, not `foo.#module-name#.x`. + remove(sess, &path); } - } else if crate_output.outputs.contains_key(&output_type) { - // 2) Multiple codegen units, with `--emit foo=some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!("ignoring emit path because multiple .{} files \ - were produced", ext)); - } else if crate_output.single_output_file.is_some() { - // 3) Multiple codegen units, with `-o some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!("ignoring -o because multiple .{} files \ - were produced", ext)); } else { - // 4) Multiple codegen units, but no explicit name. We - // just leave the `foo.0.x` files in place. - // (We don't have to do any work in this case.) + let ext = crate_output.temp_path(output_type, None) + .extension() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + + if crate_output.outputs.contains_key(&output_type) { + // 2) Multiple codegen units, with `--emit foo=some_name`. We have + // no good solution for this case, so warn the user. + sess.warn(&format!("ignoring emit path because multiple .{} files \ + were produced", ext)); + } else if crate_output.single_output_file.is_some() { + // 3) Multiple codegen units, with `-o some_name`. We have + // no good solution for this case, so warn the user. + sess.warn(&format!("ignoring -o because multiple .{} files \ + were produced", ext)); + } else { + // 4) Multiple codegen units, but no explicit name. We + // just leave the `foo.0.x` files in place. + // (We don't have to do any work in this case.) + } } }; @@ -789,17 +806,19 @@ pub fn run_passes(sess: &Session, // Copy to .bc, but always keep the .0.bc. There is a later // check to figure out if we should delete .0.bc files, or keep // them for making an rlib. - copy_if_one_unit("0.bc", OutputType::Bitcode, true); + copy_if_one_unit(OutputType::Bitcode, true); } OutputType::LlvmAssembly => { - copy_if_one_unit("0.ll", OutputType::LlvmAssembly, false); + copy_if_one_unit(OutputType::LlvmAssembly, false); } OutputType::Assembly => { - copy_if_one_unit("0.s", OutputType::Assembly, false); + // TODO: These are probably wrong + copy_if_one_unit(OutputType::Assembly, false); } OutputType::Object => { user_wants_objects = true; - copy_if_one_unit("0.o", OutputType::Object, true); + // TODO: These are probably wrong + copy_if_one_unit(OutputType::Object, true); } OutputType::Exe | OutputType::DepInfo => {} @@ -810,51 +829,55 @@ pub fn run_passes(sess: &Session, // Clean up unwanted temporary files. // We create the following files by default: - // - crate.0.bc - // - crate.0.o + // - crate.#module-name#.bc + // - crate.#module-name#.o // - crate.metadata.bc // - crate.metadata.o // - crate.o (linked from crate.##.o) - // - crate.bc (copied from crate.0.bc) + // - crate.bc (copied from crate.##.bc) // We may create additional files if requested by the user (through // `-C save-temps` or `--emit=` flags). if !sess.opts.cg.save_temps { - // Remove the temporary .0.o objects. If the user didn't + // Remove the temporary .#module-name#.o objects. If the user didn't // explicitly request bitcode (with --emit=bc), and the bitcode is not - // needed for building an rlib, then we must remove .0.bc as well. + // needed for building an rlib, then we must remove .#module-name#.bc as + // well. - // Specific rules for keeping .0.bc: + // Specific rules for keeping .#module-name#.bc: // - If we're building an rlib (`needs_crate_bitcode`), then keep // it. // - If the user requested bitcode (`user_wants_bitcode`), and // codegen_units > 1, then keep it. // - If the user requested bitcode but codegen_units == 1, then we - // can toss .0.bc because we copied it to .bc earlier. + // can toss .#module-name#.bc because we copied it to .bc earlier. // - If we're not building an rlib and the user didn't request - // bitcode, then delete .0.bc. + // bitcode, then delete .#module-name#.bc. // If you change how this works, also update back::link::link_rlib, - // where .0.bc files are (maybe) deleted after making an rlib. + // where .#module-name#.bc files are (maybe) deleted after making an + // rlib. let keep_numbered_bitcode = needs_crate_bitcode || (user_wants_bitcode && sess.opts.cg.codegen_units > 1); let keep_numbered_objects = needs_crate_object || (user_wants_objects && sess.opts.cg.codegen_units > 1); - for i in 0..trans.modules.len() { + for module_name in trans.modules.iter().map(|m| Some(&m.name[..])) { if modules_config.emit_obj && !keep_numbered_objects { - let ext = format!("{}.o", i); - remove(sess, &crate_output.with_extension(&ext)); + let path = crate_output.temp_path(OutputType::Object, module_name); + remove(sess, &path); } if modules_config.emit_bc && !keep_numbered_bitcode { - let ext = format!("{}.bc", i); - remove(sess, &crate_output.with_extension(&ext)); + let path = crate_output.temp_path(OutputType::Bitcode, module_name); + remove(sess, &path); } } if metadata_config.emit_bc && !user_wants_bitcode { - remove(sess, &crate_output.with_extension("metadata.bc")); + let path = crate_output.temp_path(OutputType::Bitcode, + Some(&trans.metadata_module.name[..])); + remove(sess, &path); } } @@ -874,28 +897,31 @@ pub fn run_passes(sess: &Session, struct WorkItem { mtrans: ModuleTranslation, config: ModuleConfig, - output_names: OutputFilenames, - name_extra: String + output_names: OutputFilenames } fn build_work_item(sess: &Session, mtrans: ModuleTranslation, config: ModuleConfig, - output_names: OutputFilenames, - name_extra: String) + output_names: OutputFilenames) -> WorkItem { let mut config = config; config.tm = create_target_machine(sess); - WorkItem { mtrans: mtrans, config: config, output_names: output_names, - name_extra: name_extra } + WorkItem { + mtrans: mtrans, + config: config, + output_names: output_names + } } fn execute_work_item(cgcx: &CodegenContext, work_item: WorkItem) { unsafe { - optimize_and_codegen(cgcx, work_item.mtrans, work_item.config, - work_item.name_extra, work_item.output_names); + optimize_and_codegen(cgcx, + work_item.mtrans, + work_item.config, + work_item.output_names); } } @@ -914,6 +940,8 @@ fn run_work_singlethreaded(sess: &Session, fn run_work_multithreaded(sess: &Session, work_items: Vec, num_workers: usize) { + assert!(num_workers > 0); + // Run some workers to process the work items. let work_items_arc = Arc::new(Mutex::new(work_items)); let mut diag_emitter = SharedEmitter::new(); @@ -981,7 +1009,7 @@ pub fn run_assembler(sess: &Session, outputs: &OutputFilenames) { let (pname, mut cmd, _) = get_linker(sess); cmd.arg("-c").arg("-o").arg(&outputs.path(OutputType::Object)) - .arg(&outputs.temp_path(OutputType::Assembly)); + .arg(&outputs.temp_path(OutputType::Assembly, None)); debug!("{:?}", cmd); match cmd.output() { diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index fc45d73a7ae..41c214fe1b3 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -2630,6 +2630,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }); let metadata_module = ModuleTranslation { + name: "metadata".to_string(), llcx: shared_ccx.metadata_llcx(), llmod: shared_ccx.metadata_llmod(), }; @@ -2644,7 +2645,11 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let crate_context_list = CrateContextList::new(&shared_ccx, codegen_units); let modules = crate_context_list.iter() - .map(|ccx| ModuleTranslation { llcx: ccx.llcx(), llmod: ccx.llmod() }) + .map(|ccx| ModuleTranslation { + name: String::from(&ccx.codegen_unit().name[..]), + llcx: ccx.llcx(), + llmod: ccx.llmod() + }) .collect(); // Skip crate items and just output metadata in -Z no-trans mode. @@ -2790,6 +2795,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } let linker_info = LinkerInfo::new(&shared_ccx, &reachable_symbols); + CrateTranslation { modules: modules, metadata_module: metadata_module, diff --git a/src/librustc_trans/lib.rs b/src/librustc_trans/lib.rs index 9cb5d8b6ad6..c465edc7311 100644 --- a/src/librustc_trans/lib.rs +++ b/src/librustc_trans/lib.rs @@ -129,8 +129,9 @@ mod type_; mod type_of; mod value; -#[derive(Copy, Clone)] +#[derive(Clone)] pub struct ModuleTranslation { + pub name: String, pub llcx: llvm::ContextRef, pub llmod: llvm::ModuleRef, } diff --git a/src/librustc_trans/partitioning.rs b/src/librustc_trans/partitioning.rs index 2ded643ef4f..d15519a87d9 100644 --- a/src/librustc_trans/partitioning.rs +++ b/src/librustc_trans/partitioning.rs @@ -121,6 +121,7 @@ use llvm; use monomorphize; use rustc::hir::def_id::DefId; use rustc::hir::map::DefPathData; +use rustc::session::config::NUMBERED_CODEGEN_UNIT_MARKER; use rustc::ty::TyCtxt; use rustc::ty::item_path::characteristic_def_id_of_type; use syntax::parse::token::{self, InternedString}; @@ -283,7 +284,10 @@ fn merge_codegen_units<'tcx>(initial_partitioning: &mut PreInliningPartitioning< } fn numbered_codegen_unit_name(crate_name: &str, index: usize) -> InternedString { - token::intern_and_get_ident(&format!("{}.{}", crate_name, index)[..]) + token::intern_and_get_ident(&format!("{}{}{}", + crate_name, + NUMBERED_CODEGEN_UNIT_MARKER, + index)[..]) } }