OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rollup merge of #60385 - nnethercote:earlier-metadata, r=alexcrichton 

Emit metadata files earlier

This will make cargo pipelining much more effective.
This commit is contained in:
Mazdak Farrokhzad 2019-05-02 01:09:26 +02:00 committed by GitHub
parent 01ce87ad14 38dffeba21
commit a7cbd92d13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 166 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock
View File

@ -2802,6 +2802,7 @@ dependencies = [
"rustc-rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_allocator 0.0.0",
"rustc_borrowck 0.0.0",
"rustc_codegen_ssa 0.0.0",
"rustc_codegen_utils 0.0.0",
"rustc_data_structures 0.0.0",
"rustc_errors 0.0.0",
@ -2821,6 +2822,7 @@ dependencies = [
"syntax 0.0.0",
"syntax_ext 0.0.0",
"syntax_pos 0.0.0",
"tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]

40
src/librustc_codegen_llvm/base.rs
View File

@ -28,7 +28,7 @@ use rustc::mir::mono::{Linkage, Visibility, Stats};
use rustc::middle::cstore::{EncodedMetadata};
use rustc::ty::TyCtxt;
use rustc::middle::exported_symbols;
use rustc::session::config::{self, DebugInfo};
use rustc::session::config::DebugInfo;
use rustc_codegen_ssa::mono_item::MonoItemExt;
use rustc_data_structures::small_c_str::SmallCStr;
@ -42,47 +42,16 @@ use rustc::hir::CodegenFnAttrs;
use crate::value::Value;
pub fn write_metadata<'a, 'gcx>(
pub fn write_compressed_metadata<'a, 'gcx>(
tcx: TyCtxt<'a, 'gcx, 'gcx>,
metadata: &EncodedMetadata,
llvm_module: &mut ModuleLlvm
) -> EncodedMetadata {
) {
use std::io::Write;
use flate2::Compression;
use flate2::write::DeflateEncoder;
let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed
}
let kind = tcx.sess.crate_types.borrow().iter().map(|ty| {
match *ty {
config::CrateType::Executable |
config::CrateType::Staticlib |
config::CrateType::Cdylib => MetadataKind::None,
config::CrateType::Rlib => MetadataKind::Uncompressed,
config::CrateType::Dylib |
config::CrateType::ProcMacro => MetadataKind::Compressed,
}
}).max().unwrap_or(MetadataKind::None);
if kind == MetadataKind::None {
return EncodedMetadata::new();
}
let metadata = tcx.encode_metadata();
if kind == MetadataKind::Uncompressed {
return metadata;
}
assert!(kind == MetadataKind::Compressed);
let mut compressed = tcx.metadata_encoding_version();
DeflateEncoder::new(&mut compressed, Compression::fast())
.write_all(&metadata.raw_data).unwrap();
@ -107,7 +76,6 @@ pub fn write_metadata<'a, 'gcx>(
let directive = CString::new(directive).unwrap();
llvm::LLVMSetModuleInlineAsm(metadata_llmod, directive.as_ptr())
}
return metadata;
}
pub struct ValueIter<'ll> {

14
src/librustc_codegen_llvm/lib.rs
View File

@ -110,12 +110,13 @@ impl ExtraBackendMethods for LlvmCodegenBackend {
ModuleLlvm::new_metadata(tcx, mod_name)
}
fn write_metadata<'b, 'gcx>(
fn write_compressed_metadata<'b, 'gcx>(
&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,
metadata: &mut ModuleLlvm
) -> EncodedMetadata {
base::write_metadata(tcx, metadata)
metadata: &EncodedMetadata,
llvm_module: &mut ModuleLlvm
) {
base::write_compressed_metadata(tcx, metadata, llvm_module)
}
fn codegen_allocator<'b, 'gcx>(
&self,
@ -289,9 +290,12 @@ impl CodegenBackend for LlvmCodegenBackend {
fn codegen_crate<'b, 'tcx>(
&self,
tcx: TyCtxt<'b, 'tcx, 'tcx>,
metadata: EncodedMetadata,
need_metadata_module: bool,
rx: mpsc::Receiver<Box<dyn Any + Send>>
) -> Box<dyn Any> {
box rustc_codegen_ssa::base::codegen_crate(LlvmCodegenBackend(()), tcx, rx)
box rustc_codegen_ssa::base::codegen_crate(
LlvmCodegenBackend(()), tcx, metadata, need_metadata_module, rx)
}
fn join_codegen_and_link(

124
src/librustc_codegen_ssa/back/link.rs
View File

@ -7,7 +7,7 @@ use rustc::session::config::{
};
use rustc::session::search_paths::PathKind;
use rustc::middle::dependency_format::Linkage;
use rustc::middle::cstore::{LibSource, NativeLibrary, NativeLibraryKind};
use rustc::middle::cstore::{EncodedMetadata, LibSource, NativeLibrary, NativeLibraryKind};
use rustc::util::common::{time, time_ext};
use rustc::hir::def_id::CrateNum;
use rustc_data_structures::fx::FxHashSet;
@ -50,9 +50,9 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
outputs: &OutputFilenames,
crate_name: &str,
target_cpu: &str) {
let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
for &crate_type in sess.crate_types.borrow().iter() {
// Ignore executable crates if we have -Z no-codegen, as they will error.
let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
if (sess.opts.debugging_opts.no_codegen || !sess.opts.output_types.should_codegen()) &&
!output_metadata &&
crate_type == config::CrateType::Executable {
@ -63,12 +63,43 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
bug!("invalid output type `{:?}` for target os `{}`",
crate_type, sess.opts.target_triple);
}
link_binary_output::<B>(sess,
codegen_results,
crate_type,
outputs,
crate_name,
target_cpu);
for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
check_file_is_writeable(obj, sess);
}
let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
sess.fatal(&format!("couldn't create a temp dir: {}", err)));
if outputs.outputs.should_codegen() {
let out_filename = out_filename(sess, crate_type, outputs, crate_name);
match crate_type {
config::CrateType::Rlib => {
link_rlib::<B>(sess,
codegen_results,
RlibFlavor::Normal,
&out_filename,
&tmpdir).build();
}
config::CrateType::Staticlib => {
link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
}
_ => {
link_natively::<B>(
sess,
crate_type,
&out_filename,
codegen_results,
tmpdir.path(),
target_cpu,
);
}
}
}
if sess.opts.cg.save_temps {
let _ = tmpdir.into_path();
}
}
// Remove the temporary object file and metadata if we aren't saving temps
@ -85,7 +116,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
if let Some(ref obj) = metadata_module.object {
remove(sess, obj);
}
}
}
if let Some(ref allocator_module) = codegen_results.allocator_module {
if let Some(ref obj) = allocator_module.object {
remove(sess, obj);
@ -97,73 +128,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
}
}
fn link_binary_output<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
codegen_results: &CodegenResults,
crate_type: config::CrateType,
outputs: &OutputFilenames,
crate_name: &str,
target_cpu: &str) {
for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
check_file_is_writeable(obj, sess);
}
if outputs.outputs.contains_key(&OutputType::Metadata) {
let out_filename = filename_for_metadata(sess, crate_name, outputs);
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with a `fs::rename` call. In order for the rename to
// always succeed, the temporary file needs to be on the same filesystem,
// which is why we create it inside the output directory specifically.
let metadata_tmpdir = TempFileBuilder::new()
.prefix("rmeta")
.tempdir_in(out_filename.parent().unwrap())
.unwrap_or_else(|err| sess.fatal(&format!("couldn't create a temp dir: {}", err)));
let metadata = emit_metadata(sess, codegen_results, &metadata_tmpdir);
match fs::rename(&metadata, &out_filename) {
Ok(_) => {
if sess.opts.debugging_opts.emit_directives {
sess.parse_sess.span_diagnostic.maybe_emit_json_directive(
format!("metadata file written: {}", out_filename.display()));
}
}
Err(e) => sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)),
}
}
let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
sess.fatal(&format!("couldn't create a temp dir: {}", err)));
if outputs.outputs.should_codegen() {
let out_filename = out_filename(sess, crate_type, outputs, crate_name);
match crate_type {
config::CrateType::Rlib => {
link_rlib::<B>(sess,
codegen_results,
RlibFlavor::Normal,
&out_filename,
&tmpdir).build();
}
config::CrateType::Staticlib => {
link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
}
_ => {
link_natively::<B>(
sess,
crate_type,
&out_filename,
codegen_results,
tmpdir.path(),
target_cpu,
);
}
}
}
if sess.opts.cg.save_temps {
let _ = tmpdir.into_path();
}
}
// The third parameter is for env vars, used on windows to set up the
// path for MSVC to find its DLLs, and gcc to find its bundled
// toolchain
@ -261,13 +225,13 @@ pub fn each_linked_rlib(sess: &Session,
/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
/// directory being searched for `extern crate` (observing an incomplete file).
/// The returned path is the temporary file containing the complete metadata.
fn emit_metadata<'a>(
pub fn emit_metadata<'a>(
sess: &'a Session,
codegen_results: &CodegenResults,
metadata: &EncodedMetadata,
tmpdir: &TempDir
) -> PathBuf {
let out_filename = tmpdir.path().join(METADATA_FILENAME);
let result = fs::write(&out_filename, &codegen_results.metadata.raw_data);
let result = fs::write(&out_filename, &metadata.raw_data);
if let Err(e) = result {
sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
@ -351,7 +315,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
RlibFlavor::Normal => {
// Instead of putting the metadata in an object file section, rlibs
// contain the metadata in a separate file.
ab.add_file(&emit_metadata(sess, codegen_results, tmpdir));
ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir));
// For LTO purposes, the bytecode of this library is also inserted
// into the archive.

48
src/librustc_codegen_ssa/base.rs
View File

@ -17,6 +17,7 @@ use crate::{ModuleCodegen, ModuleKind, CachedModuleCodegen};
use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
use rustc::hir::def_id::{DefId, LOCAL_CRATE};
use rustc::middle::cstore::EncodedMetadata;
use rustc::middle::lang_items::StartFnLangItem;
use rustc::middle::weak_lang_items;
use rustc::mir::mono::{Stats, CodegenUnitNameBuilder};
@ -25,7 +26,7 @@ use rustc::ty::layout::{self, Align, TyLayout, LayoutOf, VariantIdx, HasTyCtxt};
use rustc::ty::query::Providers;
use rustc::middle::cstore::{self, LinkagePreference};
use rustc::util::common::{time, print_time_passes_entry};
use rustc::session::config::{self, CrateType, EntryFnType, Lto};
use rustc::session::config::{self, EntryFnType, Lto};
use rustc::session::Session;
use rustc_mir::monomorphize::item::DefPathBasedNames;
use rustc_mir::monomorphize::Instance;
@ -530,26 +531,13 @@ pub const CODEGEN_WORKER_ID: usize = ::std::usize::MAX;
pub fn codegen_crate<B: ExtraBackendMethods>(
backend: B,
tcx: TyCtxt<'a, 'tcx, 'tcx>,
metadata: EncodedMetadata,
need_metadata_module: bool,
rx: mpsc::Receiver<Box<dyn Any + Send>>
) -> OngoingCodegen<B> {
check_for_rustc_errors_attr(tcx);
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
// Codegen the metadata.
tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));
let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
&["crate"],
Some("metadata")).as_str()
.to_string();
let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
let metadata = time(tcx.sess, "write metadata", || {
backend.write_metadata(tcx, &mut metadata_llvm_module)
});
tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));
// Skip crate items and just output metadata in -Z no-codegen mode.
if tcx.sess.opts.debugging_opts.no_codegen ||
!tcx.sess.opts.output_types.should_codegen() {
@ -569,6 +557,8 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
return ongoing_codegen;
}
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
// Run the monomorphization collector and partition the collected items into
// codegen units.
let codegen_units = tcx.collect_and_partition_mono_items(LOCAL_CRATE).1;
@ -632,17 +622,21 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module);
}
let needs_metadata_module = tcx.sess.crate_types.borrow().iter().any(|ct| {
match *ct {
CrateType::Dylib |
CrateType::ProcMacro => true,
CrateType::Executable |
CrateType::Rlib |
CrateType::Staticlib |
CrateType::Cdylib => false,
}
});
if needs_metadata_module {
if need_metadata_module {
// Codegen the encoded metadata.
tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));
let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
&["crate"],
Some("metadata")).as_str()
.to_string();
let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
time(tcx.sess, "write compressed metadata", || {
backend.write_compressed_metadata(tcx, &ongoing_codegen.metadata,
&mut metadata_llvm_module);
});
tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));
let metadata_module = ModuleCodegen {
name: metadata_cgu_name,
module_llvm: metadata_llvm_module,

7
src/librustc_codegen_ssa/traits/backend.rs
View File

@ -33,11 +33,12 @@ impl<'tcx, T> Backend<'tcx> for T where
pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send {
fn new_metadata(&self, sess: TyCtxt<'_, '_, '_>, mod_name: &str) -> Self::Module;
fn write_metadata<'b, 'gcx>(
fn write_compressed_metadata<'b, 'gcx>(
&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,
metadata: &mut Self::Module,
) -> EncodedMetadata;
metadata: &EncodedMetadata,
llvm_module: &mut Self::Module,
);
fn codegen_allocator<'b, 'gcx>(
&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,

4
src/librustc_codegen_utils/codegen_backend.rs
View File

@ -18,7 +18,7 @@ use rustc::util::common::ErrorReported;
use rustc::session::config::{OutputFilenames, PrintRequest};
use rustc::ty::TyCtxt;
use rustc::ty::query::Providers;
use rustc::middle::cstore::MetadataLoader;
use rustc::middle::cstore::{EncodedMetadata, MetadataLoader};
use rustc::dep_graph::DepGraph;
pub use rustc_data_structures::sync::MetadataRef;
@ -37,6 +37,8 @@ pub trait CodegenBackend {
fn codegen_crate<'a, 'tcx>(
&self,
tcx: TyCtxt<'a, 'tcx, 'tcx>,
metadata: EncodedMetadata,
need_metadata_module: bool,
rx: mpsc::Receiver<Box<dyn Any + Send>>
) -> Box<dyn Any>;

2
src/librustc_interface/Cargo.toml
View File

@ -24,6 +24,7 @@ rustc_borrowck = { path = "../librustc_borrowck" }
rustc_incremental = { path = "../librustc_incremental" }
rustc_traits = { path = "../librustc_traits" }
rustc_data_structures = { path = "../librustc_data_structures" }
rustc_codegen_ssa = { path = "../librustc_codegen_ssa" }
rustc_codegen_utils = { path = "../librustc_codegen_utils" }
rustc_metadata = { path = "../librustc_metadata" }
rustc_mir = { path = "../librustc_mir" }
@ -34,3 +35,4 @@ rustc_errors = { path = "../librustc_errors" }
rustc_plugin = { path = "../librustc_plugin" }
rustc_privacy = { path = "../librustc_privacy" }
rustc_resolve = { path = "../librustc_resolve" }
tempfile = "3.0.5"

77
src/librustc_interface/passes.rs
View File

@ -16,11 +16,13 @@ use rustc::traits;
use rustc::util::common::{time, ErrorReported};
use rustc::util::profiling::ProfileCategory;
use rustc::session::{CompileResult, CrateDisambiguator, Session};
use rustc::session::config::{self, Input, OutputFilenames, OutputType};
use rustc::session::config::{self, CrateType, Input, OutputFilenames, OutputType};
use rustc::session::search_paths::PathKind;
use rustc_allocator as allocator;
use rustc_borrowck as borrowck;
use rustc_codegen_ssa::back::link::emit_metadata;
use rustc_codegen_utils::codegen_backend::CodegenBackend;
use rustc_codegen_utils::link::filename_for_metadata;
use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::stable_hasher::StableHasher;
@ -50,6 +52,7 @@ use syntax_pos::{FileName, hygiene};
use syntax_ext;
use serialize::json;
use tempfile::Builder as TempFileBuilder;
use std::any::Any;
use std::env;
@ -999,6 +1002,68 @@ fn analysis<'tcx>(
Ok(())
}
fn encode_and_write_metadata<'tcx>(
tcx: TyCtxt<'_, 'tcx, 'tcx>,
outputs: &OutputFilenames,
) -> (middle::cstore::EncodedMetadata, bool) {
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed
}
let metadata_kind = tcx.sess.crate_types.borrow().iter().map(|ty| {
match *ty {
CrateType::Executable |
CrateType::Staticlib |
CrateType::Cdylib => MetadataKind::None,
CrateType::Rlib => MetadataKind::Uncompressed,
CrateType::Dylib |
CrateType::ProcMacro => MetadataKind::Compressed,
}
}).max().unwrap_or(MetadataKind::None);
let metadata = match metadata_kind {
MetadataKind::None => middle::cstore::EncodedMetadata::new(),
MetadataKind::Uncompressed |
MetadataKind::Compressed => tcx.encode_metadata(),
};
let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata);
if need_metadata_file {
let crate_name = &tcx.crate_name(LOCAL_CRATE).as_str();
let out_filename = filename_for_metadata(tcx.sess, crate_name, outputs);
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with an `fs::rename` call. In order for the rename to
// always succeed, the temporary file needs to be on the same filesystem,
// which is why we create it inside the output directory specifically.
let metadata_tmpdir = TempFileBuilder::new()
.prefix("rmeta")
.tempdir_in(out_filename.parent().unwrap())
.unwrap_or_else(|err| {
tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))
});
let metadata_filename = emit_metadata(tcx.sess, &metadata, &metadata_tmpdir);
match std::fs::rename(&metadata_filename, &out_filename) {
Ok(_) => {
if tcx.sess.opts.debugging_opts.emit_directives {
tcx.sess.parse_sess.span_diagnostic.maybe_emit_json_directive(
format!("metadata file written: {}", out_filename.display()));
}
}
Err(e) => tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)),
}
}
let need_metadata_module = metadata_kind == MetadataKind::Compressed;
(metadata, need_metadata_module)
}
/// Runs the codegen backend, after which the AST and analysis can
/// be discarded.
pub fn start_codegen<'tcx>(
@ -1013,11 +1078,17 @@ pub fn start_codegen<'tcx>(
}
time(tcx.sess, "resolving dependency formats", || {
::rustc::middle::dependency_format::calculate(tcx)
middle::dependency_format::calculate(tcx)
});
let (metadata, need_metadata_module) = time(tcx.sess, "metadata encoding and writing", || {
encode_and_write_metadata(tcx, outputs)
});
tcx.sess.profiler(|p| p.start_activity("codegen crate"));
let codegen = time(tcx.sess, "codegen", move || codegen_backend.codegen_crate(tcx, rx));
let codegen = time(tcx.sess, "codegen", move || {
codegen_backend.codegen_crate(tcx, metadata, need_metadata_module, rx)
});
tcx.sess.profiler(|p| p.end_activity("codegen crate"));
if log_enabled!(::log::Level::Info) {

4
src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs
View File

@ -15,7 +15,7 @@ use rustc::session::Session;
use rustc::session::config::OutputFilenames;
use rustc::ty::TyCtxt;
use rustc::ty::query::Providers;
use rustc::middle::cstore::MetadataLoader;
use rustc::middle::cstore::{EncodedMetadata, MetadataLoader};
use rustc::dep_graph::DepGraph;
use rustc::util::common::ErrorReported;
use rustc_codegen_utils::codegen_backend::CodegenBackend;
@ -61,6 +61,8 @@ impl CodegenBackend for TheBackend {
fn codegen_crate<'a, 'tcx>(
&self,
tcx: TyCtxt<'a, 'tcx, 'tcx>,
_metadata: EncodedMetadata,
_need_metadata_module: bool,
_rx: mpsc::Receiver<Box<Any + Send>>
) -> Box<Any> {
use rustc::hir::def_id::LOCAL_CRATE;