From abd5cc3364b9ec9500ef8f5e64699e25069bc51b Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 31 Aug 2018 15:18:08 +0200 Subject: [PATCH] Always add all modules to the global ThinLTO module analysis when compiling incrementally. --- src/Cargo.lock | 11 + src/librustc/dep_graph/graph.rs | 1 - src/librustc_codegen_llvm/back/lto.rs | 239 ++++++++---------- src/librustc_codegen_llvm/back/write.rs | 154 ++--------- src/librustc_codegen_llvm/base.rs | 129 +++------- .../persist/work_product.rs | 1 - 6 files changed, 172 insertions(+), 363 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index e4d9f6e2394..defb5b9869d 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1198,6 +1198,15 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memmap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "memoffset" version = "0.2.1" @@ -2029,6 +2038,7 @@ name = "rustc_codegen_llvm" version = "0.0.0" dependencies = [ "cc 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_llvm 0.0.0", @@ -3151,6 +3161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum mdbook 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "90b5a8d7e341ceee5db3882a06078d42661ddcfa2b3687319cc5da76ec4e782f" "checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d" +"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum minifier 0.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "9908ed7c62f990c21ab41fdca53a864a3ada0da69d8729c4de727b397e27bc11" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 1c883eeed96..4df0fc443a2 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -883,7 +883,6 @@ pub enum WorkProductFileKind { Object, Bytecode, BytecodeCompressed, - PreThinLtoBytecode, } pub(super) struct CurrentDepGraph { diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 98131349927..6936a306780 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -11,17 +11,18 @@ use back::bytecode::{DecodedBytecode, RLIB_BYTECODE_EXTENSION}; use back::symbol_export; use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext}; -use back::write::{self, DiagnosticHandlers}; +use back::write::{self, DiagnosticHandlers, pre_lto_bitcode_filename}; use errors::{FatalError, Handler}; use llvm::archive_ro::ArchiveRO; use llvm::{True, False}; use llvm; use memmap; +use rustc::dep_graph::WorkProduct; use rustc::hir::def_id::LOCAL_CRATE; use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashMap; use time_graph::Timeline; use {ModuleCodegen, ModuleLlvm, ModuleKind}; @@ -29,15 +30,10 @@ use libc; use std::ffi::{CStr, CString}; use std::fs::File; -use std::io; -use std::mem; -use std::path::Path; use std::ptr; use std::slice; use std::sync::Arc; -pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin"; - pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { match crate_type { config::CrateType::Executable | @@ -105,11 +101,16 @@ impl LtoModuleCodegen { } } +/// Performs LTO, which in the case of full LTO means merging all modules into +/// a single one and returning it for further optimizing. For ThinLTO, it will +/// do the global analysis necessary and return two lists, one of the modules +/// the need optimization and another for modules that can simply be copied over +/// from the incr. comp. cache. pub(crate) fn run(cgcx: &CodegenContext, modules: Vec, - import_only_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, timeline: &mut Timeline) - -> Result, FatalError> + -> Result<(Vec, Vec), FatalError> { let diag_handler = cgcx.create_diag_handler(); let export_threshold = match cgcx.lto { @@ -202,13 +203,14 @@ pub(crate) fn run(cgcx: &CodegenContext, match cgcx.lto { Lto::Yes | // `-C lto` == fat LTO by default Lto::Fat => { - assert!(import_only_modules.is_empty()); - fat_lto(cgcx, - &diag_handler, - modules, - upstream_modules, - &symbol_white_list, - timeline) + assert!(cached_modules.is_empty()); + let opt_jobs = fat_lto(cgcx, + &diag_handler, + modules, + upstream_modules, + &symbol_white_list, + timeline); + opt_jobs.map(|opt_jobs| (opt_jobs, vec![])) } Lto::Thin | Lto::ThinLocal => { @@ -220,7 +222,7 @@ pub(crate) fn run(cgcx: &CodegenContext, &diag_handler, modules, upstream_modules, - import_only_modules, + cached_modules, &symbol_white_list, timeline) } @@ -388,14 +390,19 @@ fn thin_lto(cgcx: &CodegenContext, diag_handler: &Handler, modules: Vec, serialized_modules: Vec<(SerializedModule, CString)>, - import_only_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, symbol_white_list: &[*const libc::c_char], timeline: &mut Timeline) - -> Result, FatalError> + -> Result<(Vec, Vec), FatalError> { unsafe { info!("going for that thin, thin LTO"); + let green_modules: FxHashMap<_, _> = cached_modules + .iter() + .map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())) + .collect(); + let mut thin_buffers = Vec::new(); let mut module_names = Vec::new(); let mut thin_modules = Vec::new(); @@ -411,6 +418,28 @@ fn thin_lto(cgcx: &CodegenContext, info!("local module: {} - {}", i, module.name); let name = CString::new(module.name.clone()).unwrap(); let buffer = ThinBuffer::new(module.module_llvm.llmod()); + + // We emit the module after having serialized it into a ThinBuffer + // because only then it will contain the ThinLTO module summary. + if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { + if cgcx.config(module.kind).emit_pre_thin_lto_bc { + use std::io::Write; + + let path = incr_comp_session_dir + .join(pre_lto_bitcode_filename(&module.name)); + let mut file = File::create(&path).unwrap_or_else(|e| { + panic!("Failed to create pre-lto-bitcode file `{}`: {}", + path.display(), + e); + }); + file.write_all(buffer.data()).unwrap_or_else(|e| { + panic!("Error writing pre-lto-bitcode file `{}`: {}", + path.display(), + e); + }); + } + } + thin_modules.push(llvm::ThinLTOModule { identifier: name.as_ptr(), data: buffer.data().as_ptr(), @@ -438,8 +467,13 @@ fn thin_lto(cgcx: &CodegenContext, // looking at upstream modules entirely sometimes (the contents, // we must always unconditionally look at the index). let mut serialized = Vec::new(); - for (module, name) in serialized_modules { - info!("foreign module {:?}", name); + + let cached_modules = cached_modules.into_iter().map(|(sm, wp)| { + (sm, CString::new(wp.cgu_name).unwrap()) + }); + + for (module, name) in serialized_modules.into_iter().chain(cached_modules) { + info!("upstream or cached module {:?}", name); thin_modules.push(llvm::ThinLTOModule { identifier: name.as_ptr(), data: module.data().as_ptr(), @@ -449,21 +483,8 @@ fn thin_lto(cgcx: &CodegenContext, module_names.push(name); } - // All the modules collected up to this point we actually want to - // optimize. The `import_only_modules` below need to be in the list of - // available modules but we don't need to run optimizations for them - // since we already have their optimized version cached. - let modules_to_optimize = module_names.len(); - for (module, name) in import_only_modules { - info!("foreign module {:?}", name); - thin_modules.push(llvm::ThinLTOModule { - identifier: name.as_ptr(), - data: module.data().as_ptr(), - len: module.data().len(), - }); - serialized.push(module); - module_names.push(name); - } + // Sanity check + assert_eq!(thin_modules.len(), module_names.len()); // Delegate to the C++ bindings to create some data here. Once this is a // tried-and-true interface we may wish to try to upstream some of this @@ -478,30 +499,7 @@ fn thin_lto(cgcx: &CodegenContext, write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string()) })?; - // Save the ThinLTO import information for incremental compilation. - if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { - let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); - - // The import information from the current compilation session. It - // does not contain info about modules that have been loaded from - // the cache instead of having been recompiled... - let current_imports = ThinLTOImports::from_thin_lto_data(data); - - // ... so we load this additional information from the previous - // cache file if necessary. - let imports = if path.exists() { - let prev_imports = ThinLTOImports::load_from_file(&path).unwrap(); - prev_imports.update(current_imports, &module_names) - } else { - current_imports - }; - - if let Err(err) = imports.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO import data: {}", - err); - return Err(write::llvm_err(&diag_handler, msg)); - } - } + let import_map = ThinLTOImports::from_thin_lto_data(data); let data = ThinData(data); info!("thin LTO data created"); @@ -517,12 +515,36 @@ fn thin_lto(cgcx: &CodegenContext, serialized_modules: serialized, module_names, }); - Ok((0..modules_to_optimize).map(|i| { - LtoModuleCodegen::Thin(ThinModule { + + let mut copy_jobs = vec![]; + let mut opt_jobs = vec![]; + + for (module_index, module_name) in shared.module_names.iter().enumerate() { + let module_name = module_name_to_str(module_name); + + if green_modules.contains_key(module_name) { + let mut imports_all_green = true; + for imported_module in import_map.modules_imported_by(module_name) { + if !green_modules.contains_key(imported_module) { + imports_all_green = false; + break + } + } + + if imports_all_green { + let work_product = green_modules[module_name].clone(); + copy_jobs.push(work_product); + continue + } + } + + opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { shared: shared.clone(), - idx: i, - }) - }).collect()) + idx: module_index, + })); + } + + Ok((opt_jobs, copy_jobs)) } } @@ -850,44 +872,12 @@ pub struct ThinLTOImports { } impl ThinLTOImports { - pub fn new() -> ThinLTOImports { - ThinLTOImports { - imports: FxHashMap(), - } - } - pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) } - pub fn update(mut self, new: ThinLTOImports, module_names: &[CString]) -> ThinLTOImports { - let module_names: FxHashSet<_> = module_names.iter().map(|name| { - name.clone().into_string().unwrap() - }).collect(); - - // Remove all modules that don't exist anymore. - self.imports.retain(|k, _| module_names.contains(k)); - - // Overwrite old values - for (importing_module, imported_modules) in new.imports { - self.imports.insert(importing_module, imported_modules); - } - - self - } - /// Load the ThinLTO import map from ThinLTOData. unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { - fn module_name_to_str(c_str: &CStr) -> &str { - match c_str.to_str() { - Ok(s) => s, - Err(e) => { - bug!("Encountered non-utf8 LLVM module name `{}`: {}", - c_str.to_string_lossy(), - e) - } - } - } unsafe extern "C" fn imported_module_callback(payload: *mut libc::c_void, importing_module_name: *const libc::c_char, imported_module_name: *const libc::c_char) { @@ -896,6 +886,7 @@ impl ThinLTOImports { let importing_module_name = module_name_to_str(&importing_module_name); let imported_module_name = CStr::from_ptr(imported_module_name); let imported_module_name = module_name_to_str(&imported_module_name); + if !map.imports.contains_key(importing_module_name) { map.imports.insert(importing_module_name.to_owned(), vec![]); } @@ -913,47 +904,15 @@ impl ThinLTOImports { &mut map as *mut _ as *mut libc::c_void); map } - - pub fn save_to_file(&self, path: &Path) -> io::Result<()> { - use std::io::Write; - let file = File::create(path)?; - let mut writer = io::BufWriter::new(file); - for (importing_module_name, imported_modules) in &self.imports { - writeln!(writer, "{}", importing_module_name)?; - for imported_module in imported_modules { - writeln!(writer, " {}", imported_module)?; - } - writeln!(writer)?; - } - Ok(()) - } - - pub fn load_from_file(path: &Path) -> io::Result { - use std::io::BufRead; - let mut imports = FxHashMap(); - let mut current_module = None; - let mut current_imports = vec![]; - let file = File::open(path)?; - for line in io::BufReader::new(file).lines() { - let line = line?; - if line.is_empty() { - let importing_module = current_module - .take() - .expect("Importing module not set"); - imports.insert(importing_module, - mem::replace(&mut current_imports, vec![])); - } else if line.starts_with(" ") { - // This is an imported module - assert_ne!(current_module, None); - current_imports.push(line.trim().to_string()); - } else { - // This is the beginning of a new module - assert_eq!(current_module, None); - current_module = Some(line.trim().to_string()); - } - } - Ok(ThinLTOImports { - imports - }) - } } + +fn module_name_to_str(c_str: &CStr) -> &str { + match c_str.to_str() { + Ok(s) => s, + Err(e) => { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", + c_str.to_string_lossy(), + e) + } + } +} \ No newline at end of file diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index e1d69db83b9..9c0f4c39790 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -28,7 +28,7 @@ use rustc::util::nodemap::FxHashMap; use time_graph::{self, TimeGraph, Timeline}; use llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; use llvm_util; -use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, ModuleLlvm, +use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, // ModuleLlvm, CachedModuleCodegen}; use CrateInfo; use rustc::hir::def_id::{CrateNum, LOCAL_CRATE}; @@ -228,8 +228,8 @@ pub struct ModuleConfig { pgo_use: String, // Flags indicating which outputs to produce. + pub emit_pre_thin_lto_bc: bool, emit_no_opt_bc: bool, - emit_pre_thin_lto_bc: bool, emit_bc: bool, emit_bc_compressed: bool, emit_lto_bc: bool, @@ -625,20 +625,13 @@ unsafe fn optimize(cgcx: &CodegenContext, // Deallocate managers that we're now done with llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); - - if config.emit_pre_thin_lto_bc { - let out = cgcx.output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, - module_name); - let out = path2cstr(&out); - llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); - } } Ok(()) } fn generate_lto_work(cgcx: &CodegenContext, modules: Vec, - import_only_modules: Vec<(SerializedModule, CString)>) + import_only_modules: Vec<(SerializedModule, WorkProduct)>) -> Vec<(WorkItem, u64)> { let mut timeline = cgcx.time_graph.as_ref().map(|tg| { @@ -646,13 +639,22 @@ fn generate_lto_work(cgcx: &CodegenContext, CODEGEN_WORK_PACKAGE_KIND, "generate lto") }).unwrap_or(Timeline::noop()); - let lto_modules = lto::run(cgcx, modules, import_only_modules, &mut timeline) + let (lto_modules, copy_jobs) = lto::run(cgcx, modules, import_only_modules, &mut timeline) .unwrap_or_else(|e| e.raise()); - lto_modules.into_iter().map(|module| { + let lto_modules = lto_modules.into_iter().map(|module| { let cost = module.cost(); (WorkItem::LTO(module), cost) - }).collect() + }); + + let copy_jobs = copy_jobs.into_iter().map(|wp| { + (WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { + name: wp.cgu_name.clone(), + source: wp, + }), 0) + }); + + lto_modules.chain(copy_jobs).collect() } unsafe fn codegen(cgcx: &CodegenContext, @@ -1083,7 +1085,6 @@ pub fn start_async_codegen(tcx: TyCtxt, fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, compiled_modules: &CompiledModules, - output_filenames: &OutputFilenames, ) -> FxHashMap { let mut work_products = FxHashMap::default(); @@ -1104,13 +1105,6 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( files.push((WorkProductFileKind::BytecodeCompressed, path.clone())); } - let pre_thin_lto_bytecode_path = - output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, Some(&module.name)); - - if pre_thin_lto_bytecode_path.exists() { - files.push((WorkProductFileKind::PreThinLtoBytecode, pre_thin_lto_bytecode_path)); - } - if let Some((id, product)) = copy_cgu_workproducts_to_incr_comp_cache_dir(sess, &module.name, &files) { work_products.insert(id, product); @@ -1285,9 +1279,6 @@ enum WorkItem { /// Copy the post-LTO artifacts from the incremental cache to the output /// directory. CopyPostLtoArtifacts(CachedModuleCodegen), - /// Load the pre-LTO version of a module from the incremental cache, so it - /// can be run through LTO again. - LoadPreLtoModule(CachedModuleCodegen), /// Perform (Thin)LTO on the given module. LTO(lto::LtoModuleCodegen), } @@ -1297,7 +1288,6 @@ impl WorkItem { match *self { WorkItem::Optimize(ref m) => m.kind, WorkItem::CopyPostLtoArtifacts(_) | - WorkItem::LoadPreLtoModule(_) | WorkItem::LTO(_) => ModuleKind::Regular, } } @@ -1305,7 +1295,6 @@ impl WorkItem { fn name(&self) -> String { match *self { WorkItem::Optimize(ref m) => format!("optimize: {}", m.name), - WorkItem::LoadPreLtoModule(ref m) => format!("load pre-lto module: {}", m.name), WorkItem::CopyPostLtoArtifacts(ref m) => format!("copy post LTO artifacts: {}", m.name), WorkItem::LTO(ref m) => format!("lto: {}", m.name()), } @@ -1326,9 +1315,6 @@ fn execute_work_item(cgcx: &CodegenContext, work_item @ WorkItem::Optimize(_) => { execute_optimize_work_item(cgcx, work_item, timeline) } - work_item @ WorkItem::LoadPreLtoModule(_) => { - execute_load_pre_lto_mod_work_item(cgcx, work_item, timeline) - } work_item @ WorkItem::CopyPostLtoArtifacts(_) => { execute_copy_from_cache_work_item(cgcx, work_item, timeline) } @@ -1454,9 +1440,6 @@ fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, bytecode_compressed = Some(path.clone()); path } - WorkProductFileKind::PreThinLtoBytecode => { - continue; - } }; let source_file = in_incr_comp_dir(&incr_comp_session_dir, &saved_file); @@ -1509,69 +1492,6 @@ fn execute_lto_work_item(cgcx: &CodegenContext, } } -fn execute_load_pre_lto_mod_work_item(cgcx: &CodegenContext, - work_item: WorkItem, - _: &mut Timeline) - -> Result -{ - let module = if let WorkItem::LoadPreLtoModule(module) = work_item { - module - } else { - bug!("execute_load_pre_lto_mod_work_item() called with wrong WorkItem kind.") - }; - - let work_product = module.source.clone(); - let incr_comp_session_dir = cgcx.incr_comp_session_dir - .as_ref() - .unwrap(); - - let filename = pre_lto_bitcode_filename(&work_product); - let bc_path = in_incr_comp_dir(&incr_comp_session_dir, &filename); - - let file = fs::File::open(&bc_path).unwrap_or_else(|e| { - panic!("failed to open bitcode file `{}`: {}", - bc_path.display(), - e); - }); - - let module_llvm = unsafe { - let data = ::memmap::Mmap::map(&file).unwrap_or_else(|e| { - panic!("failed to create mmap for bitcode file `{}`: {}", - bc_path.display(), - e); - }); - - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let mod_name_c = SmallCStr::new(&module.name); - let llmod_raw = match llvm::LLVMRustParseBitcodeForThinLTO( - llcx, - data.as_ptr(), - data.len(), - mod_name_c.as_ptr(), - ) { - Some(m) => m as *const _, - None => { - panic!("failed to parse bitcode for thin LTO module `{}`", - module.name); - } - }; - - let tm = (cgcx.tm_factory)().unwrap(); - - ModuleLlvm { - llmod_raw, - llcx, - tm, - } - }; - - Ok(WorkItemResult::NeedsLTO(ModuleCodegen { - name: module.name.to_string(), - module_llvm, - kind: ModuleKind::Regular, - })) -} - enum Message { Token(io::Result), NeedsLTO { @@ -1588,7 +1508,7 @@ enum Message { }, AddImportOnlyModule { module_data: SerializedModule, - module_name: CString, + work_product: WorkProduct, }, CodegenComplete, CodegenItem, @@ -1893,6 +1813,7 @@ fn start_executing_work(tcx: TyCtxt, work_items.len() > 0 || running > 0 || needs_lto.len() > 0 || + lto_import_only_modules.len() > 0 || main_thread_worker_state != MainThreadWorkerState::Idle { // While there are still CGUs to be codegened, the coordinator has @@ -1932,7 +1853,7 @@ fn start_executing_work(tcx: TyCtxt, running == 0 && main_thread_worker_state == MainThreadWorkerState::Idle { assert!(!started_lto); - assert!(needs_lto.len() > 0); + assert!(needs_lto.len() + lto_import_only_modules.len() > 0); started_lto = true; let modules = mem::replace(&mut needs_lto, Vec::new()); let import_only_modules = @@ -2104,10 +2025,13 @@ fn start_executing_work(tcx: TyCtxt, free_worker_ids.push(worker_id); needs_lto.push(result); } - Message::AddImportOnlyModule { module_data, module_name } => { + Message::AddImportOnlyModule { module_data, work_product } => { assert!(!started_lto); assert!(!codegen_done); - lto_import_only_modules.push((module_data, module_name)); + assert_eq!(main_thread_worker_state, + MainThreadWorkerState::Codegenning); + lto_import_only_modules.push((module_data, work_product)); + main_thread_worker_state = MainThreadWorkerState::Idle; } Message::Done { result: Err(()), worker_id: _ } => { shared_emitter.fatal("aborting due to worker thread failure"); @@ -2483,8 +2407,7 @@ impl OngoingCodegen { let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, - &compiled_modules, - &self.output_filenames); + &compiled_modules); produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); @@ -2565,19 +2488,7 @@ pub(crate) fn submit_post_lto_module_to_llvm(tcx: TyCtxt, pub(crate) fn submit_pre_lto_module_to_llvm(tcx: TyCtxt, module: CachedModuleCodegen) { - let llvm_work_item = WorkItem::LoadPreLtoModule(module); - - drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { - llvm_work_item, - // We don't know the size of the module, but just loading will have smaller - // cost than optimizing. - cost: 10, - }))); -} - -pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt, - module: CachedModuleCodegen) { - let filename = pre_lto_bitcode_filename(&module.source); + let filename = pre_lto_bitcode_filename(&module.name); let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); let file = fs::File::open(&bc_path).unwrap_or_else(|e| { panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e) @@ -2592,21 +2503,12 @@ pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt, // Schedule the module to be loaded drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::AddImportOnlyModule { module_data: SerializedModule::FromUncompressedFile(mmap, file), - module_name: CString::new(module.name.clone()).unwrap(), + work_product: module.source, }))); - - // Note: We also schedule for the cached files to be copied to the output - // directory - submit_post_lto_module_to_llvm(tcx, module); } -fn pre_lto_bitcode_filename(wp: &WorkProduct) -> String { - wp.saved_files - .iter() - .find(|&&(kind, _)| kind == WorkProductFileKind::PreThinLtoBytecode) - .map(|&(_, ref filename)| filename.clone()) - .unwrap_or_else(|| panic!("Couldn't find pre-thin-lto bytecode for `{}`", - wp.cgu_name)) +pub(super) fn pre_lto_bitcode_filename(module_name: &str) -> String { + format!("{}.{}", module_name, PRE_THIN_LTO_BC_EXT) } fn msvc_imps_needed(tcx: TyCtxt) -> bool { diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 10e415190b4..c1f6006e684 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -29,7 +29,6 @@ use super::ModuleKind; use super::CachedModuleCodegen; use abi; -use back::lto; use back::write::{self, OngoingCodegen}; use llvm::{self, TypeKind, get_param}; use metadata; @@ -45,7 +44,7 @@ use rustc::middle::cstore::{self, LinkagePreference}; use rustc::middle::exported_symbols; use rustc::util::common::{time, print_time_passes_entry}; use rustc::util::profiling::ProfileCategory; -use rustc::session::config::{self, DebugInfo, EntryFnType}; +use rustc::session::config::{self, DebugInfo, EntryFnType, Lto}; use rustc::session::Session; use rustc_incremental; use allocator; @@ -698,77 +697,48 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> { } } -#[derive(Debug, PartialEq)] +#[derive(Debug)] enum CguReUsable { - No, - PreThinLto, - PostThinLto, - PostThinLtoButImportedFrom, + PreLto, + PostLto, + No } fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - codegen_units: &[Arc>]) - -> FxHashMap { + cgu: &CodegenUnit<'tcx>) + -> CguReUsable { if !tcx.dep_graph.is_fully_enabled() { - return codegen_units.iter() - .map(|cgu| (cgu.name().clone(), CguReUsable::No)) - .collect(); + return CguReUsable::No } - let thin_lto_imports = load_thin_lto_imports(tcx.sess); + let work_product_id = &cgu.work_product_id(); + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. + return CguReUsable::No + } - let mut reusable_cgus = FxHashMap(); - let mut green_cgus = FxHashMap(); - let mut need_for_importing = FxHashSet(); + // Try to mark the CGU as green. If it we can do so, it means that nothing + // affecting the LLVM module has changed and we can re-use a cached version. + // If we compile with any kind of LTO, this means we can re-use the bitcode + // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only + // know that later). If we are not doing LTO, there is only one optimized + // version of each module, so we re-use that. + let dep_node = cgu.codegen_dep_node(tcx); + assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name()); - for cgu in codegen_units { - let work_product_id = &cgu.work_product_id(); - if tcx.dep_graph.previous_work_product(work_product_id).is_none() { - // We don't have anything cached for this CGU. This can happen - // if the CGU did not exist in the previous session. - reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); - continue - }; - // Try to mark the CGU as green - let dep_node = cgu.codegen_dep_node(tcx); - assert!(!tcx.dep_graph.dep_node_exists(&dep_node), - "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", - cgu.name()); - - if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { - // We can re-use either the pre- or the post-thinlto state - green_cgus.insert(cgu.name().to_string(), cgu); + if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + // We can re-use either the pre- or the post-thinlto state + if tcx.sess.lto() != Lto::No { + CguReUsable::PreLto } else { - // We definitely cannot re-use this CGU - reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); - - let imported_cgus = thin_lto_imports.modules_imported_by(&cgu.name().as_str()); - need_for_importing.extend(imported_cgus.iter().cloned()); + CguReUsable::PostLto } + } else { + CguReUsable::No } - - // Now we know all CGUs that have not changed themselves. Next we need to - // check if anything they imported via ThinLTO has changed. - for (cgu_name, cgu) in &green_cgus { - let imported_cgus = thin_lto_imports.modules_imported_by(cgu_name); - let all_imports_green = imported_cgus.iter().all(|imported_cgu| { - green_cgus.contains_key(&imported_cgu[..]) - }); - if all_imports_green { - reusable_cgus.insert(cgu.name().clone(), CguReUsable::PostThinLto); - } else { - reusable_cgus.insert(cgu.name().clone(), CguReUsable::PreThinLto); - need_for_importing.extend(imported_cgus.iter().cloned()); - } - } - - for (name, state) in reusable_cgus.iter_mut() { - if *state == CguReUsable::PostThinLto && need_for_importing.contains(&name.as_str()[..]) { - *state = CguReUsable::PostThinLtoButImportedFrom; - } - } - - reusable_cgus } pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, @@ -920,13 +890,11 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let mut total_codegen_time = Duration::new(0, 0); let mut all_stats = Stats::default(); - let cgu_reuse = determine_cgu_reuse(tcx, &codegen_units); - for cgu in codegen_units.into_iter() { ongoing_codegen.wait_for_signal_to_codegen_item(); ongoing_codegen.check_for_errors(tcx.sess); - let loaded_from_cache = match cgu_reuse[cgu.name()] { + let loaded_from_cache = match determine_cgu_reuse(tcx, &cgu) { CguReUsable::No => { let _timing_guard = time_graph.as_ref().map(|time_graph| { time_graph.start(write::CODEGEN_WORKER_TIMELINE, @@ -939,21 +907,14 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, total_codegen_time += start_time.elapsed(); false } - CguReUsable::PreThinLto => { + CguReUsable::PreLto => { write::submit_pre_lto_module_to_llvm(tcx, CachedModuleCodegen { name: cgu.name().to_string(), source: cgu.work_product(tcx), }); true } - CguReUsable::PostThinLtoButImportedFrom => { - write::submit_import_only_module_to_llvm(tcx, CachedModuleCodegen { - name: cgu.name().to_string(), - source: cgu.work_product(tcx), - }); - true - } - CguReUsable::PostThinLto => { + CguReUsable::PostLto => { write::submit_post_lto_module_to_llvm(tcx, CachedModuleCodegen { name: cgu.name().to_string(), source: cgu.work_product(tcx), @@ -1391,28 +1352,6 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { } } -fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { - if sess.opts.incremental.is_none() { - return lto::ThinLTOImports::new(); - } - - let path = rustc_incremental::in_incr_comp_dir_sess( - sess, - lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME - ); - if !path.exists() { - return lto::ThinLTOImports::new(); - } - match lto::ThinLTOImports::load_from_file(&path) { - Ok(imports) => imports, - Err(e) => { - let msg = format!("Error while trying to load ThinLTO import data \ - for incremental compilation: {}", e); - sess.fatal(&msg) - } - } -} - // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement // the HashStable trait. Normally DepGraph::with_task() calls are // hidden behind queries, but CGU creation is a special case in two diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs index c285934d75b..cfe59b1f672 100644 --- a/src/librustc_incremental/persist/work_product.rs +++ b/src/librustc_incremental/persist/work_product.rs @@ -36,7 +36,6 @@ pub fn copy_cgu_workproducts_to_incr_comp_cache_dir( WorkProductFileKind::Object => "o", WorkProductFileKind::Bytecode => "bc", WorkProductFileKind::BytecodeCompressed => "bc.z", - WorkProductFileKind::PreThinLtoBytecode => "pre-thinlto.bc", }; let file_name = format!("{}.{}", cgu_name, extension); let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name);