From d129ac2655451ca894bceeb58a4d5c26a86c4caa Mon Sep 17 00:00:00 2001 From: Rich Kadel Date: Sat, 15 Aug 2020 04:42:13 -0700 Subject: [PATCH] Moved coverage counter injection from BasicBlock to Statement. --- library/core/src/intrinsics.rs | 64 --- src/librustc_codegen_llvm/builder.rs | 6 +- .../coverageinfo/mapgen.rs | 20 +- src/librustc_codegen_llvm/coverageinfo/mod.rs | 36 +- src/librustc_codegen_llvm/intrinsic.rs | 103 ----- src/librustc_codegen_ssa/coverageinfo/ffi.rs | 2 +- src/librustc_codegen_ssa/coverageinfo/map.rs | 131 ++---- src/librustc_codegen_ssa/coverageinfo/mod.rs | 3 - src/librustc_codegen_ssa/mir/block.rs | 12 - src/librustc_codegen_ssa/mir/coverageinfo.rs | 35 ++ src/librustc_codegen_ssa/mir/mod.rs | 1 + src/librustc_codegen_ssa/mir/statement.rs | 4 + src/librustc_codegen_ssa/traits/builder.rs | 2 +- .../traits/coverageinfo.rs | 18 +- src/librustc_codegen_ssa/traits/intrinsic.rs | 14 +- src/librustc_hir/fake_lang_items.rs | 37 -- src/librustc_hir/lang_items.rs | 5 - src/librustc_hir/lib.rs | 1 - src/librustc_middle/mir/coverage/mod.rs | 125 ++++-- src/librustc_middle/mir/mod.rs | 15 +- src/librustc_middle/mir/visit.rs | 19 + src/librustc_middle/ty/structural_impls.rs | 5 + src/librustc_mir/borrow_check/def_use.rs | 3 +- src/librustc_mir/borrow_check/invalidation.rs | 1 + src/librustc_mir/borrow_check/mod.rs | 1 + .../borrow_check/type_check/mod.rs | 1 + src/librustc_mir/dataflow/impls/borrows.rs | 1 + .../dataflow/impls/init_locals.rs | 1 + .../dataflow/impls/storage_liveness.rs | 1 + .../dataflow/move_paths/builder.rs | 1 + src/librustc_mir/interpret/intrinsics.rs | 5 - src/librustc_mir/interpret/step.rs | 13 + .../transform/check_consts/validation.rs | 1 + src/librustc_mir/transform/check_unsafety.rs | 1 + src/librustc_mir/transform/generator.rs | 1 + .../transform/instrument_coverage.rs | 380 +++++------------- .../transform/qualify_min_const_fn.rs | 1 + .../transform/remove_noop_landing_pads.rs | 1 + src/librustc_passes/weak_lang_items.rs | 14 +- src/librustc_span/symbol.rs | 4 - src/librustc_typeck/check/intrinsic.rs | 41 -- ...ument_coverage.bar.InstrumentCoverage.diff | 24 +- ...ment_coverage.main.InstrumentCoverage.diff | 23 +- src/test/mir-opt/instrument_coverage.rs | 4 +- 44 files changed, 397 insertions(+), 784 deletions(-) create mode 100644 src/librustc_codegen_ssa/mir/coverageinfo.rs delete mode 100644 src/librustc_hir/fake_lang_items.rs diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 3a28bc79eff..1399d0c020f 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1949,70 +1949,6 @@ extern "rust-intrinsic" { #[rustc_const_unstable(feature = "const_ptr_offset_from", issue = "41079")] pub fn ptr_offset_from(ptr: *const T, base: *const T) -> isize; - /// Internal placeholder for injecting code coverage counters when the "instrument-coverage" - /// option is enabled. The source code region information is extracted prior to code generation, - /// and added to the "coverage map", which is injected into the generated code as additional - /// data. This intrinsic then triggers the generation of LLVM intrinsic call - /// `instrprof.increment`, using the remaining args (`function_source_hash` and `index`). - #[cfg(not(bootstrap))] - #[lang = "count_code_region"] - pub fn count_code_region( - function_source_hash: u64, - index: u32, - file_name: &'static str, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ); - - /// Internal marker for code coverage expressions, injected into the MIR when the - /// "instrument-coverage" option is enabled. This intrinsic is not converted into a - /// backend intrinsic call, but its arguments are extracted during the production of a - /// "coverage map", which is injected into the generated code, as additional data. - /// This marker identifies a code region and two other counters or counter expressions - /// whose sum is the number of times the code region was executed. - #[cfg(not(bootstrap))] - #[lang = "coverage_counter_add"] - pub fn coverage_counter_add( - index: u32, - left_index: u32, - right_index: u32, - file_name: &'static str, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ); - - /// This marker identifies a code region and two other counters or counter expressions - /// whose difference is the number of times the code region was executed. - /// (See `coverage_counter_add` for more information.) - #[cfg(not(bootstrap))] - #[lang = "coverage_counter_subtract"] - pub fn coverage_counter_subtract( - index: u32, - left_index: u32, - right_index: u32, - file_name: &'static str, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ); - - /// This marker identifies a code region to be added to the "coverage map" to indicate source - /// code that can never be reached. - /// (See `coverage_counter_add` for more information.) - #[cfg(not(bootstrap))] - pub fn coverage_unreachable( - file_name: &'static str, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ); - /// See documentation of `<*const T>::guaranteed_eq` for details. #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] pub fn ptr_guaranteed_eq(ptr: *const T, other: *const T) -> bool; diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs index 4737c73d57f..4ece08f6293 100644 --- a/src/librustc_codegen_llvm/builder.rs +++ b/src/librustc_codegen_llvm/builder.rs @@ -1117,7 +1117,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { hash: &'ll Value, num_counters: &'ll Value, index: &'ll Value, - ) -> &'ll Value { + ) { debug!( "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", fn_name, hash, num_counters, index @@ -1128,13 +1128,13 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("call", llfn, args); unsafe { - llvm::LLVMRustBuildCall( + let _ = llvm::LLVMRustBuildCall( self.llbuilder, llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, None, - ) + ); } } diff --git a/src/librustc_codegen_llvm/coverageinfo/mapgen.rs b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs index a39234c0b7f..ec6c177614d 100644 --- a/src/librustc_codegen_llvm/coverageinfo/mapgen.rs +++ b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs @@ -3,14 +3,16 @@ use crate::coverageinfo; use crate::llvm; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, Region}; +use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods}; use rustc_data_structures::fx::FxIndexSet; use rustc_llvm::RustString; -use tracing::debug; +use rustc_middle::mir::coverage::CodeRegion; use std::ffi::CString; +use tracing::debug; + /// Generates and exports the Coverage Map. /// /// This Coverage Map complies with Coverage Mapping Format version 3 (zero-based encoded as 2), @@ -91,7 +93,7 @@ impl CoverageMapGenerator { fn write_coverage_mappings( &mut self, expressions: Vec, - counter_regions: impl Iterator)>, + counter_regions: impl Iterator, coverage_mappings_buffer: &RustString, ) { let mut counter_regions = counter_regions.collect::>(); @@ -104,22 +106,22 @@ impl CoverageMapGenerator { let mut current_file_name = None; let mut current_file_id = 0; - // Convert the list of (Counter, Region) pairs to an array of `CounterMappingRegion`, sorted + // Convert the list of (Counter, CodeRegion) pairs to an array of `CounterMappingRegion`, sorted // by filename and position. Capture any new files to compute the `CounterMappingRegion`s // `file_id` (indexing files referenced by the current function), and construct the // function-specific `virtual_file_mapping` from `file_id` to its index in the module's // `filenames` array. counter_regions.sort_unstable_by_key(|(_counter, region)| *region); for (counter, region) in counter_regions { - let Region { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.as_ref().map_or(false, |p| p == file_name); + let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; + let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); if !same_file { if current_file_name.is_some() { current_file_id += 1; } - current_file_name = Some(file_name.to_string()); - let c_filename = - CString::new(file_name).expect("null error converting filename to C string"); + current_file_name = Some(file_name); + let c_filename = CString::new(file_name.to_string()) + .expect("null error converting filename to C string"); debug!(" file_id: {} = '{:?}'", current_file_id, c_filename); let (filenames_index, _) = self.filenames.insert_full(c_filename); virtual_file_mapping.push(filenames_index as u32); diff --git a/src/librustc_codegen_llvm/coverageinfo/mod.rs b/src/librustc_codegen_llvm/coverageinfo/mod.rs index 54570920219..2bd37bf9c4f 100644 --- a/src/librustc_codegen_llvm/coverageinfo/mod.rs +++ b/src/librustc_codegen_llvm/coverageinfo/mod.rs @@ -5,18 +5,22 @@ use crate::common::CodegenCx; use libc::c_uint; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, ExprKind, FunctionCoverage, Region}; +use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; use rustc_codegen_ssa::traits::{ BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, MiscMethods, StaticMethods, }; use rustc_data_structures::fx::FxHashMap; use rustc_llvm::RustString; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionIndex, Op, +}; use rustc_middle::ty::Instance; -use tracing::debug; use std::cell::RefCell; use std::ffi::CString; +use tracing::debug; + pub mod mapgen; const COVMAP_VAR_ALIGN_BYTES: usize = 8; @@ -24,7 +28,7 @@ const COVMAP_VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'tcx> { // Coverage region data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: RefCell, FunctionCoverage<'tcx>>>, + pub(crate) function_coverage_map: RefCell, FunctionCoverage>>, } impl<'tcx> CrateCoverageContext<'tcx> { @@ -32,7 +36,7 @@ impl<'tcx> CrateCoverageContext<'tcx> { Self { function_coverage_map: Default::default() } } - pub fn take_function_coverage_map(&self) -> FxHashMap, FunctionCoverage<'tcx>> { + pub fn take_function_coverage_map(&self) -> FxHashMap, FunctionCoverage> { self.function_coverage_map.replace(FxHashMap::default()) } } @@ -58,11 +62,11 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { &mut self, instance: Instance<'tcx>, function_source_hash: u64, - id: u32, - region: Region<'tcx>, + id: CounterValueReference, + region: CodeRegion, ) { debug!( - "adding counter to coverage_regions: instance={:?}, function_source_hash={}, id={}, \ + "adding counter to coverage_regions: instance={:?}, function_source_hash={}, id={:?}, \ at {:?}", instance, function_source_hash, id, region, ); @@ -76,25 +80,25 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { fn add_counter_expression_region( &mut self, instance: Instance<'tcx>, - id_descending_from_max: u32, - lhs: u32, - op: ExprKind, - rhs: u32, - region: Region<'tcx>, + id: InjectedExpressionIndex, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: CodeRegion, ) { debug!( - "adding counter expression to coverage_regions: instance={:?}, id={}, {} {:?} {}, \ + "adding counter expression to coverage_regions: instance={:?}, id={:?}, {:?} {:?} {:?}, \ at {:?}", - instance, id_descending_from_max, lhs, op, rhs, region, + instance, id, lhs, op, rhs, region, ); let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); coverage_regions .entry(instance) .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter_expression(id_descending_from_max, lhs, op, rhs, region); + .add_counter_expression(id, lhs, op, rhs, region); } - fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: Region<'tcx>) { + fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion) { debug!( "adding unreachable code to coverage_regions: instance={:?}, at {:?}", instance, region, diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index f1977946ee9..bb79a52dcf9 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -11,22 +11,18 @@ use rustc_ast as ast; use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh}; use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; -use rustc_codegen_ssa::coverageinfo; use rustc_codegen_ssa::glue; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_hir as hir; -use rustc_middle::mir::coverage; -use rustc_middle::mir::Operand; use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; use rustc_target::abi::{self, HasDataLayout, LayoutOf, Primitive}; use rustc_target::spec::PanicStrategy; -use tracing::debug; use std::cmp::Ordering; use std::iter; @@ -83,77 +79,6 @@ fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<&'ll Va } impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { - fn is_codegen_intrinsic( - &mut self, - intrinsic: Symbol, - args: &Vec>, - caller_instance: ty::Instance<'tcx>, - ) -> bool { - let mut is_codegen_intrinsic = true; - // Set `is_codegen_intrinsic` to `false` to bypass `codegen_intrinsic_call()`. - - // FIXME(richkadel): Make sure to add coverage analysis tests on a crate with - // external crate dependencies, where: - // 1. Both binary and dependent crates are compiled with `-Zinstrument-coverage` - // 2. Only binary is compiled with `-Zinstrument-coverage` - // 3. Only dependent crates are compiled with `-Zinstrument-coverage` - match intrinsic { - sym::count_code_region => { - use coverage::count_code_region_args::*; - self.add_counter_region( - caller_instance, - op_to_u64(&args[FUNCTION_SOURCE_HASH]), - op_to_u32(&args[COUNTER_ID]), - coverageinfo::Region::new( - op_to_str_slice(&args[FILE_NAME]), - op_to_u32(&args[START_LINE]), - op_to_u32(&args[START_COL]), - op_to_u32(&args[END_LINE]), - op_to_u32(&args[END_COL]), - ), - ); - } - sym::coverage_counter_add | sym::coverage_counter_subtract => { - is_codegen_intrinsic = false; - use coverage::coverage_counter_expression_args::*; - self.add_counter_expression_region( - caller_instance, - op_to_u32(&args[EXPRESSION_ID]), - op_to_u32(&args[LEFT_ID]), - if intrinsic == sym::coverage_counter_add { - coverageinfo::ExprKind::Add - } else { - coverageinfo::ExprKind::Subtract - }, - op_to_u32(&args[RIGHT_ID]), - coverageinfo::Region::new( - op_to_str_slice(&args[FILE_NAME]), - op_to_u32(&args[START_LINE]), - op_to_u32(&args[START_COL]), - op_to_u32(&args[END_LINE]), - op_to_u32(&args[END_COL]), - ), - ); - } - sym::coverage_unreachable => { - is_codegen_intrinsic = false; - use coverage::coverage_unreachable_args::*; - self.add_unreachable_region( - caller_instance, - coverageinfo::Region::new( - op_to_str_slice(&args[FILE_NAME]), - op_to_u32(&args[START_LINE]), - op_to_u32(&args[START_COL]), - op_to_u32(&args[END_LINE]), - op_to_u32(&args[END_COL]), - ), - ); - } - _ => {} - } - is_codegen_intrinsic - } - fn codegen_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, @@ -161,7 +86,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { args: &[OperandRef<'tcx, &'ll Value>], llresult: &'ll Value, span: Span, - caller_instance: ty::Instance<'tcx>, ) { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); @@ -213,21 +137,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let llfn = self.get_intrinsic(&("llvm.debugtrap")); self.call(llfn, &[], None) } - sym::count_code_region => { - use coverage::count_code_region_args::*; - let coverageinfo = tcx.coverageinfo(caller_instance.def_id()); - - let fn_name = self.create_pgo_func_name_var(caller_instance); - let hash = args[FUNCTION_SOURCE_HASH].immediate(); - let num_counters = self.const_u32(coverageinfo.num_counters); - let index = args[COUNTER_ID].immediate(); - debug!( - "translating Rust intrinsic `count_code_region()` to LLVM intrinsic: \ - instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, index, - ); - self.instrprof_increment(fn_name, hash, num_counters, index) - } sym::va_start => self.va_start(args[0].immediate()), sym::va_end => self.va_end(args[0].immediate()), sym::va_copy => { @@ -2238,15 +2147,3 @@ fn float_type_width(ty: Ty<'_>) -> Option { _ => None, } } - -fn op_to_str_slice<'tcx>(op: &Operand<'tcx>) -> &'tcx str { - Operand::value_from_const(op).try_to_str_slice().expect("Value is &str") -} - -fn op_to_u32<'tcx>(op: &Operand<'tcx>) -> u32 { - Operand::scalar_from_const(op).to_u32().expect("Scalar is u32") -} - -fn op_to_u64<'tcx>(op: &Operand<'tcx>) -> u64 { - Operand::scalar_from_const(op).to_u64().expect("Scalar is u64") -} diff --git a/src/librustc_codegen_ssa/coverageinfo/ffi.rs b/src/librustc_codegen_ssa/coverageinfo/ffi.rs index 5b04f994994..a266d179a42 100644 --- a/src/librustc_codegen_ssa/coverageinfo/ffi.rs +++ b/src/librustc_codegen_ssa/coverageinfo/ffi.rs @@ -1,4 +1,4 @@ -use super::map::{CounterValueReference, MappedExpressionIndex}; +use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; /// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/10.0-2020-05-05/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L91) #[derive(Copy, Clone, Debug)] diff --git a/src/librustc_codegen_ssa/coverageinfo/map.rs b/src/librustc_codegen_ssa/coverageinfo/map.rs index 7f6841f9daa..814e43c5fa5 100644 --- a/src/librustc_codegen_ssa/coverageinfo/map.rs +++ b/src/librustc_codegen_ssa/coverageinfo/map.rs @@ -1,66 +1,19 @@ pub use super::ffi::*; use rustc_index::vec::IndexVec; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionIndex, + MappedExpressionIndex, Op, +}; use rustc_middle::ty::Instance; use rustc_middle::ty::TyCtxt; -use std::cmp::Ord; - -rustc_index::newtype_index! { - pub struct ExpressionOperandId { - DEBUG_FORMAT = "ExpressionOperandId({})", - MAX = 0xFFFF_FFFF, - } -} - -rustc_index::newtype_index! { - pub struct CounterValueReference { - DEBUG_FORMAT = "CounterValueReference({})", - MAX = 0xFFFF_FFFF, - } -} - -rustc_index::newtype_index! { - pub struct InjectedExpressionIndex { - DEBUG_FORMAT = "InjectedExpressionIndex({})", - MAX = 0xFFFF_FFFF, - } -} - -rustc_index::newtype_index! { - pub struct MappedExpressionIndex { - DEBUG_FORMAT = "MappedExpressionIndex({})", - MAX = 0xFFFF_FFFF, - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct Region<'tcx> { - pub file_name: &'tcx str, - pub start_line: u32, - pub start_col: u32, - pub end_line: u32, - pub end_col: u32, -} - -impl<'tcx> Region<'tcx> { - pub fn new( - file_name: &'tcx str, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { file_name, start_line, start_col, end_line, end_col } - } -} - #[derive(Clone, Debug)] -pub struct ExpressionRegion<'tcx> { +pub struct ExpressionRegion { lhs: ExpressionOperandId, - op: ExprKind, + op: Op, rhs: ExpressionOperandId, - region: Region<'tcx>, + region: CodeRegion, } /// Collects all of the coverage regions associated with (a) injected counters, (b) counter @@ -75,15 +28,15 @@ pub struct ExpressionRegion<'tcx> { /// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count /// for a gap area is only used as the line execution count if there are no other regions on a /// line." -pub struct FunctionCoverage<'tcx> { +pub struct FunctionCoverage { source_hash: u64, - counters: IndexVec>>, - expressions: IndexVec>>, - unreachable_regions: Vec>, + counters: IndexVec>, + expressions: IndexVec>, + unreachable_regions: Vec, } -impl<'tcx> FunctionCoverage<'tcx> { - pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { +impl FunctionCoverage { + pub fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { let coverageinfo = tcx.coverageinfo(instance.def_id()); Self { source_hash: 0, // will be set with the first `add_counter()` @@ -96,15 +49,13 @@ impl<'tcx> FunctionCoverage<'tcx> { /// Adds a code region to be counted by an injected counter intrinsic. /// The source_hash (computed during coverage instrumentation) should also be provided, and /// should be the same for all counters in a given function. - pub fn add_counter(&mut self, source_hash: u64, id: u32, region: Region<'tcx>) { + pub fn add_counter(&mut self, source_hash: u64, id: CounterValueReference, region: CodeRegion) { if self.source_hash == 0 { self.source_hash = source_hash; } else { debug_assert_eq!(source_hash, self.source_hash); } - self.counters[CounterValueReference::from(id)] - .replace(region) - .expect_none("add_counter called with duplicate `id`"); + self.counters[id].replace(region).expect_none("add_counter called with duplicate `id`"); } /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other @@ -123,24 +74,20 @@ impl<'tcx> FunctionCoverage<'tcx> { /// counters and expressions have been added. pub fn add_counter_expression( &mut self, - id_descending_from_max: u32, - lhs: u32, - op: ExprKind, - rhs: u32, - region: Region<'tcx>, + expression_id: InjectedExpressionIndex, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: CodeRegion, ) { - let expression_id = ExpressionOperandId::from(id_descending_from_max); - let lhs = ExpressionOperandId::from(lhs); - let rhs = ExpressionOperandId::from(rhs); - - let expression_index = self.expression_index(expression_id); + let expression_index = self.expression_index(u32::from(expression_id)); self.expressions[expression_index] .replace(ExpressionRegion { lhs, op, rhs, region }) .expect_none("add_counter_expression called with duplicate `id_descending_from_max`"); } /// Add a region that will be marked as "unreachable", with a constant "zero counter". - pub fn add_unreachable_region(&mut self, region: Region<'tcx>) { + pub fn add_unreachable_region(&mut self, region: CodeRegion) { self.unreachable_regions.push(region) } @@ -153,9 +100,9 @@ impl<'tcx> FunctionCoverage<'tcx> { /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions( - &'tcx self, - ) -> (Vec, impl Iterator)>) { + pub fn get_expressions_and_counter_regions<'a>( + &'a self, + ) -> (Vec, impl Iterator) { assert!(self.source_hash != 0); let counter_regions = self.counter_regions(); @@ -167,7 +114,7 @@ impl<'tcx> FunctionCoverage<'tcx> { (counter_expressions, counter_regions) } - fn counter_regions(&'tcx self) -> impl Iterator)> { + fn counter_regions<'a>(&'a self) -> impl Iterator { self.counters.iter_enumerated().filter_map(|(index, entry)| { // Option::map() will return None to filter out missing counters. This may happen // if, for example, a MIR-instrumented counter is removed during an optimization. @@ -178,8 +125,8 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn expressions_with_regions( - &'tcx self, - ) -> (Vec, impl Iterator)>) { + &'a self, + ) -> (Vec, impl Iterator) { let mut counter_expressions = Vec::with_capacity(self.expressions.len()); let mut expression_regions = Vec::with_capacity(self.expressions.len()); let mut new_indexes = @@ -204,7 +151,7 @@ impl<'tcx> FunctionCoverage<'tcx> { .as_ref() .map(|_| Counter::counter_value_reference(index)) } else { - let index = self.expression_index(id); + let index = self.expression_index(u32::from(id)); self.expressions .get(index) .expect("expression id is out of range") @@ -232,7 +179,14 @@ impl<'tcx> FunctionCoverage<'tcx> { // been assigned a `new_index`. let mapped_expression_index = MappedExpressionIndex::from(counter_expressions.len()); - counter_expressions.push(CounterExpression::new(lhs_counter, op, rhs_counter)); + counter_expressions.push(CounterExpression::new( + lhs_counter, + match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + rhs_counter, + )); new_indexes[original_index] = mapped_expression_index; expression_regions.push((Counter::expression(mapped_expression_index), region)); } @@ -240,15 +194,12 @@ impl<'tcx> FunctionCoverage<'tcx> { (counter_expressions, expression_regions.into_iter()) } - fn unreachable_regions(&'tcx self) -> impl Iterator)> { + fn unreachable_regions<'a>(&'a self) -> impl Iterator { self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) } - fn expression_index( - &self, - id_descending_from_max: ExpressionOperandId, - ) -> InjectedExpressionIndex { - debug_assert!(id_descending_from_max.index() >= self.counters.len()); - InjectedExpressionIndex::from(u32::MAX - u32::from(id_descending_from_max)) + fn expression_index(&self, id_descending_from_max: u32) -> InjectedExpressionIndex { + debug_assert!(id_descending_from_max >= self.counters.len() as u32); + InjectedExpressionIndex::from(u32::MAX - id_descending_from_max) } } diff --git a/src/librustc_codegen_ssa/coverageinfo/mod.rs b/src/librustc_codegen_ssa/coverageinfo/mod.rs index ff794a75c36..569fd3f1a51 100644 --- a/src/librustc_codegen_ssa/coverageinfo/mod.rs +++ b/src/librustc_codegen_ssa/coverageinfo/mod.rs @@ -1,5 +1,2 @@ pub mod ffi; pub mod map; - -pub use map::ExprKind; -pub use map::Region; diff --git a/src/librustc_codegen_ssa/mir/block.rs b/src/librustc_codegen_ssa/mir/block.rs index 371e1274d8d..a4e039de4df 100644 --- a/src/librustc_codegen_ssa/mir/block.rs +++ b/src/librustc_codegen_ssa/mir/block.rs @@ -647,17 +647,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) { let intrinsic = intrinsic.unwrap(); - - // `is_codegen_intrinsic()` allows the backend implementation to perform compile-time - // operations before converting the `args` to backend values. - if !bx.is_codegen_intrinsic(intrinsic, &args, self.instance) { - // If the intrinsic call was fully addressed by the `is_codegen_intrinsic()` call - // (as a compile-time operation), return immediately. This avoids the need to - // convert the arguments, the call to `codegen_intrinsic_call()`, and the return - // value handling. - return; - } - let dest = match ret_dest { _ if fn_abi.ret.is_indirect() => llargs[0], ReturnDest::Nothing => { @@ -702,7 +691,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &args, dest, terminator.source_info.span, - self.instance, ); if let ReturnDest::IndirectOperand(dst, _) = ret_dest { diff --git a/src/librustc_codegen_ssa/mir/coverageinfo.rs b/src/librustc_codegen_ssa/mir/coverageinfo.rs new file mode 100644 index 00000000000..a2ad27b925c --- /dev/null +++ b/src/librustc_codegen_ssa/mir/coverageinfo.rs @@ -0,0 +1,35 @@ +use crate::traits::*; + +use rustc_middle::mir::coverage::*; +use rustc_middle::mir::Coverage; + +use super::FunctionCx; + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage) { + let Coverage { kind, code_region } = coverage; + match kind { + CoverageKind::Counter { function_source_hash, id } => { + bx.add_counter_region(self.instance, function_source_hash, id, code_region); + + let coverageinfo = bx.tcx().coverageinfo(self.instance.def_id()); + + let fn_name = bx.create_pgo_func_name_var(self.instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let id = bx.const_u32(u32::from(id)); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, id, + ); + bx.instrprof_increment(fn_name, hash, num_counters, id); + } + CoverageKind::Expression { id, lhs, op, rhs } => { + bx.add_counter_expression_region(self.instance, id, lhs, op, rhs, code_region); + } + CoverageKind::Unreachable => { + bx.add_unreachable_region(self.instance, code_region); + } + } + } +} diff --git a/src/librustc_codegen_ssa/mir/mod.rs b/src/librustc_codegen_ssa/mir/mod.rs index f4d83e87710..26e6c354702 100644 --- a/src/librustc_codegen_ssa/mir/mod.rs +++ b/src/librustc_codegen_ssa/mir/mod.rs @@ -484,6 +484,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( mod analyze; mod block; pub mod constant; +pub mod coverageinfo; pub mod debuginfo; pub mod operand; pub mod place; diff --git a/src/librustc_codegen_ssa/mir/statement.rs b/src/librustc_codegen_ssa/mir/statement.rs index ddd7447406c..6f74ba77d4c 100644 --- a/src/librustc_codegen_ssa/mir/statement.rs +++ b/src/librustc_codegen_ssa/mir/statement.rs @@ -111,6 +111,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } bx } + mir::StatementKind::Coverage(box ref coverage) => { + self.codegen_coverage(&mut bx, coverage.clone()); + bx + } mir::StatementKind::FakeRead(..) | mir::StatementKind::Retag { .. } | mir::StatementKind::AscribeUserType(..) diff --git a/src/librustc_codegen_ssa/traits/builder.rs b/src/librustc_codegen_ssa/traits/builder.rs index 4e11ef5fd6e..5ffc83c5f99 100644 --- a/src/librustc_codegen_ssa/traits/builder.rs +++ b/src/librustc_codegen_ssa/traits/builder.rs @@ -271,7 +271,7 @@ pub trait BuilderMethods<'a, 'tcx>: hash: Self::Value, num_counters: Self::Value, index: Self::Value, - ) -> Self::Value; + ); fn call( &mut self, diff --git a/src/librustc_codegen_ssa/traits/coverageinfo.rs b/src/librustc_codegen_ssa/traits/coverageinfo.rs index 5602599b0c2..b74e4e45901 100644 --- a/src/librustc_codegen_ssa/traits/coverageinfo.rs +++ b/src/librustc_codegen_ssa/traits/coverageinfo.rs @@ -1,5 +1,5 @@ use super::BackendTypes; -use crate::coverageinfo::{ExprKind, Region}; +use rustc_middle::mir::coverage::*; use rustc_middle::ty::Instance; pub trait CoverageInfoMethods: BackendTypes { @@ -13,19 +13,19 @@ pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { &mut self, instance: Instance<'tcx>, function_source_hash: u64, - index: u32, - region: Region<'tcx>, + id: CounterValueReference, + region: CodeRegion, ); fn add_counter_expression_region( &mut self, instance: Instance<'tcx>, - index: u32, - lhs: u32, - op: ExprKind, - rhs: u32, - region: Region<'tcx>, + id: InjectedExpressionIndex, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: CodeRegion, ); - fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: Region<'tcx>); + fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion); } diff --git a/src/librustc_codegen_ssa/traits/intrinsic.rs b/src/librustc_codegen_ssa/traits/intrinsic.rs index 425bea4cb19..9d48e233de6 100644 --- a/src/librustc_codegen_ssa/traits/intrinsic.rs +++ b/src/librustc_codegen_ssa/traits/intrinsic.rs @@ -1,8 +1,7 @@ use super::BackendTypes; use crate::mir::operand::OperandRef; -use rustc_middle::mir::Operand; use rustc_middle::ty::{self, Ty}; -use rustc_span::{Span, Symbol}; +use rustc_span::Span; use rustc_target::abi::call::FnAbi; pub trait IntrinsicCallMethods<'tcx>: BackendTypes { @@ -16,19 +15,8 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes { args: &[OperandRef<'tcx, Self::Value>], llresult: Self::Value, span: Span, - caller_instance: ty::Instance<'tcx>, ); - /// Intrinsic-specific pre-codegen processing, if any is required. Some intrinsics are handled - /// at compile time and do not generate code. Returns true if codegen is required or false if - /// the intrinsic does not need code generation. - fn is_codegen_intrinsic( - &mut self, - intrinsic: Symbol, - args: &Vec>, - caller_instance: ty::Instance<'tcx>, - ) -> bool; - fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; diff --git a/src/librustc_hir/fake_lang_items.rs b/src/librustc_hir/fake_lang_items.rs deleted file mode 100644 index 91db58054b6..00000000000 --- a/src/librustc_hir/fake_lang_items.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! Validity checking for fake lang items - -use crate::def_id::DefId; -use crate::{lang_items, LangItem, LanguageItems}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_span::symbol::{sym, Symbol}; - -use lazy_static::lazy_static; - -macro_rules! fake_lang_items { - ($($item:ident, $name:ident, $method:ident;)*) => ( - -lazy_static! { - pub static ref FAKE_ITEMS_REFS: FxHashMap = { - let mut map = FxHashMap::default(); - $(map.insert(sym::$name, lang_items::$item);)* - map - }; -} - -impl LanguageItems { - pub fn is_fake_lang_item(&self, item_def_id: DefId) -> bool { - let did = Some(item_def_id); - - $(self.$method() == did)||* - } -} - -) } - -fake_lang_items! { -// Variant name, Symbol, Method name, - CountCodeRegionFnLangItem, count_code_region, count_code_region_fn; - CoverageCounterAddFnLangItem, coverage_counter_add, coverage_counter_add_fn; - CoverageCounterSubtractFnLangItem, coverage_counter_subtract, coverage_counter_subtract_fn; -} diff --git a/src/librustc_hir/lang_items.rs b/src/librustc_hir/lang_items.rs index c00d861350d..978f73760ec 100644 --- a/src/librustc_hir/lang_items.rs +++ b/src/librustc_hir/lang_items.rs @@ -303,11 +303,6 @@ language_item_table! { TryTraitLangItem, kw::Try, try_trait, Target::Trait; - // language items related to source code coverage instrumentation (-Zinstrument-coverage) - CountCodeRegionFnLangItem, sym::count_code_region, count_code_region_fn, Target::Fn; - CoverageCounterAddFnLangItem, sym::coverage_counter_add, coverage_counter_add_fn, Target::Fn; - CoverageCounterSubtractFnLangItem, sym::coverage_counter_subtract, coverage_counter_subtract_fn, Target::Fn; - // Language items from AST lowering TryFromError, sym::from_error, from_error_fn, Target::Method(MethodKind::Trait { body: false }); TryFromOk, sym::from_ok, from_ok_fn, Target::Method(MethodKind::Trait { body: false }); diff --git a/src/librustc_hir/lib.rs b/src/librustc_hir/lib.rs index a64565b20e7..19ea1de5683 100644 --- a/src/librustc_hir/lib.rs +++ b/src/librustc_hir/lib.rs @@ -19,7 +19,6 @@ mod arena; pub mod def; pub mod definitions; pub use rustc_span::def_id; -pub mod fake_lang_items; mod hir; pub mod hir_id; pub mod intravisit; diff --git a/src/librustc_middle/mir/coverage/mod.rs b/src/librustc_middle/mir/coverage/mod.rs index 6b1514da644..ce311c2ee52 100644 --- a/src/librustc_middle/mir/coverage/mod.rs +++ b/src/librustc_middle/mir/coverage/mod.rs @@ -1,34 +1,105 @@ //! Metadata from source code coverage analysis and instrumentation. -/// Positional arguments to `libcore::count_code_region()` -pub mod count_code_region_args { - pub const FUNCTION_SOURCE_HASH: usize = 0; - pub const COUNTER_ID: usize = 1; - pub const FILE_NAME: usize = 2; - pub const START_LINE: usize = 3; - pub const START_COL: usize = 4; - pub const END_LINE: usize = 5; - pub const END_COL: usize = 6; +use rustc_macros::HashStable; +use rustc_span::Symbol; + +use std::cmp::Ord; +use std::fmt::{self, Debug, Formatter}; + +rustc_index::newtype_index! { + pub struct ExpressionOperandId { + derive [HashStable] + DEBUG_FORMAT = "ExpressionOperandId({})", + MAX = 0xFFFF_FFFF, + } } -/// Positional arguments to `libcore::coverage_counter_add()` and -/// `libcore::coverage_counter_subtract()` -pub mod coverage_counter_expression_args { - pub const EXPRESSION_ID: usize = 0; - pub const LEFT_ID: usize = 1; - pub const RIGHT_ID: usize = 2; - pub const FILE_NAME: usize = 3; - pub const START_LINE: usize = 4; - pub const START_COL: usize = 5; - pub const END_LINE: usize = 6; - pub const END_COL: usize = 7; +rustc_index::newtype_index! { + pub struct CounterValueReference { + derive [HashStable] + DEBUG_FORMAT = "CounterValueReference({})", + MAX = 0xFFFF_FFFF, + } } -/// Positional arguments to `libcore::coverage_unreachable()` -pub mod coverage_unreachable_args { - pub const FILE_NAME: usize = 0; - pub const START_LINE: usize = 1; - pub const START_COL: usize = 2; - pub const END_LINE: usize = 3; - pub const END_COL: usize = 4; +rustc_index::newtype_index! { + pub struct InjectedExpressionIndex { + derive [HashStable] + DEBUG_FORMAT = "InjectedExpressionIndex({})", + MAX = 0xFFFF_FFFF, + } +} + +rustc_index::newtype_index! { + pub struct MappedExpressionIndex { + derive [HashStable] + DEBUG_FORMAT = "MappedExpressionIndex({})", + MAX = 0xFFFF_FFFF, + } +} + +impl From for ExpressionOperandId { + #[inline] + fn from(v: CounterValueReference) -> ExpressionOperandId { + ExpressionOperandId::from(v.as_u32()) + } +} + +impl From for ExpressionOperandId { + #[inline] + fn from(v: InjectedExpressionIndex) -> ExpressionOperandId { + ExpressionOperandId::from(v.as_u32()) + } +} + +#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, HashStable, TypeFoldable)] +pub enum CoverageKind { + Counter { + function_source_hash: u64, + id: CounterValueReference, + }, + Expression { + id: InjectedExpressionIndex, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + }, + Unreachable, +} + +impl CoverageKind { + pub fn as_operand_id(&self) -> ExpressionOperandId { + match *self { + CoverageKind::Counter { id, .. } => ExpressionOperandId::from(id), + CoverageKind::Expression { id, .. } => ExpressionOperandId::from(id), + CoverageKind::Unreachable => { + bug!("Unreachable coverage cannot be part of an expression") + } + } + } +} + +#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, PartialEq, Eq, PartialOrd, Ord)] +pub struct CodeRegion { + pub file_name: Symbol, + pub start_line: u32, + pub start_col: u32, + pub end_line: u32, + pub end_col: u32, +} + +impl Debug for CodeRegion { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + write!( + fmt, + "{}:{}:{} - {}:{}", + self.file_name, self.start_line, self.start_col, self.end_line, self.end_col + ) + } +} + +#[derive(Copy, Clone, Debug, PartialEq, TyEncodable, TyDecodable, HashStable, TypeFoldable)] +pub enum Op { + Subtract, + Add, } diff --git a/src/librustc_middle/mir/mod.rs b/src/librustc_middle/mir/mod.rs index 1faed87f8ff..9cce8ee2c5d 100644 --- a/src/librustc_middle/mir/mod.rs +++ b/src/librustc_middle/mir/mod.rs @@ -2,6 +2,7 @@ //! //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html +use crate::mir::coverage::{CodeRegion, CoverageKind}; use crate::mir::interpret::{Allocation, ConstValue, GlobalAlloc, Scalar}; use crate::mir::visit::MirVisitable; use crate::ty::adjustment::PointerCast; @@ -25,7 +26,6 @@ use rustc_data_structures::graph::dominators::{dominators, Dominators}; use rustc_data_structures::graph::{self, GraphSuccessors}; use rustc_index::bit_set::BitMatrix; use rustc_index::vec::{Idx, IndexVec}; -use rustc_macros::HashStable; use rustc_serialize::{Decodable, Encodable}; use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; @@ -1400,6 +1400,12 @@ pub enum StatementKind<'tcx> { /// - `Bivariant` -- no effect AscribeUserType(Box<(Place<'tcx>, UserTypeProjection)>, ty::Variance), + /// Marks the start of a "coverage region", injected with '-Zinstrument-coverage'. A + /// `CoverageInfo` statement carries metadata about the coverage region, used to inject a coverage + /// map into the binary. The `Counter` kind also generates executable code, to increment a + /// counter varible at runtime, each time the code region is executed. + Coverage(Box), + /// No-op. Useful for deleting instructions without affecting statement indices. Nop, } @@ -1495,11 +1501,18 @@ impl Debug for Statement<'_> { AscribeUserType(box (ref place, ref c_ty), ref variance) => { write!(fmt, "AscribeUserType({:?}, {:?}, {:?})", place, variance, c_ty) } + Coverage(box ref coverage) => write!(fmt, "{:?}", coverage), Nop => write!(fmt, "nop"), } } } +#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, HashStable, TypeFoldable)] +pub struct Coverage { + pub kind: CoverageKind, + pub code_region: CodeRegion, +} + /////////////////////////////////////////////////////////////////////////// // Places diff --git a/src/librustc_middle/mir/visit.rs b/src/librustc_middle/mir/visit.rs index c6ace5bbf66..6515ae31b46 100644 --- a/src/librustc_middle/mir/visit.rs +++ b/src/librustc_middle/mir/visit.rs @@ -134,6 +134,12 @@ macro_rules! make_mir_visitor { self.super_ascribe_user_ty(place, variance, user_ty, location); } + fn visit_coverage(&mut self, + coverage: & $($mutability)? Coverage, + location: Location) { + self.super_coverage(coverage, location); + } + fn visit_retag(&mut self, kind: & $($mutability)? RetagKind, place: & $($mutability)? Place<'tcx>, @@ -389,6 +395,12 @@ macro_rules! make_mir_visitor { ) => { self.visit_ascribe_user_ty(place, variance, user_ty, location); } + StatementKind::Coverage(coverage) => { + self.visit_coverage( + coverage, + location + ) + } StatementKind::Nop => {} } } @@ -739,6 +751,11 @@ macro_rules! make_mir_visitor { self.visit_user_type_projection(user_ty); } + fn super_coverage(&mut self, + _kind: & $($mutability)? Coverage, + _location: Location) { + } + fn super_retag(&mut self, _kind: & $($mutability)? RetagKind, place: & $($mutability)? Place<'tcx>, @@ -1133,6 +1150,8 @@ pub enum NonUseContext { StorageDead, /// User type annotation assertions for NLL. AscribeUserTy, + /// Coverage code region and counter metadata. + Coverage, /// The data of an user variable, for debug info. VarDebugInfo, } diff --git a/src/librustc_middle/ty/structural_impls.rs b/src/librustc_middle/ty/structural_impls.rs index ffa5df6ef2d..605e3545dea 100644 --- a/src/librustc_middle/ty/structural_impls.rs +++ b/src/librustc_middle/ty/structural_impls.rs @@ -271,6 +271,7 @@ CloneTypeFoldableAndLiftImpls! { bool, usize, ::rustc_target::abi::VariantIdx, + u32, u64, String, crate::middle::region::Scope, @@ -289,6 +290,10 @@ CloneTypeFoldableAndLiftImpls! { ::rustc_hir::Unsafety, ::rustc_target::asm::InlineAsmRegOrRegClass, ::rustc_target::spec::abi::Abi, + crate::mir::coverage::ExpressionOperandId, + crate::mir::coverage::CounterValueReference, + crate::mir::coverage::InjectedExpressionIndex, + crate::mir::coverage::MappedExpressionIndex, crate::mir::Local, crate::mir::Promoted, crate::traits::Reveal, diff --git a/src/librustc_mir/borrow_check/def_use.rs b/src/librustc_mir/borrow_check/def_use.rs index 689ec249a2f..6574e584406 100644 --- a/src/librustc_mir/borrow_check/def_use.rs +++ b/src/librustc_mir/borrow_check/def_use.rs @@ -72,7 +72,8 @@ pub fn categorize(context: PlaceContext) -> Option { PlaceContext::MutatingUse(MutatingUseContext::Drop) => Some(DefUse::Drop), - // Debug info is neither def nor use. + // Coverage and debug info are neither def nor use. + PlaceContext::NonUse(NonUseContext::Coverage) | PlaceContext::NonUse(NonUseContext::VarDebugInfo) => None, } } diff --git a/src/librustc_mir/borrow_check/invalidation.rs b/src/librustc_mir/borrow_check/invalidation.rs index 2de2124dc5e..c84ccafaff5 100644 --- a/src/librustc_mir/borrow_check/invalidation.rs +++ b/src/librustc_mir/borrow_check/invalidation.rs @@ -93,6 +93,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> { } } StatementKind::Nop + | StatementKind::Coverage(..) | StatementKind::AscribeUserType(..) | StatementKind::Retag { .. } | StatementKind::StorageLive(..) => { diff --git a/src/librustc_mir/borrow_check/mod.rs b/src/librustc_mir/borrow_check/mod.rs index f7031b2a598..9a28321a6cc 100644 --- a/src/librustc_mir/borrow_check/mod.rs +++ b/src/librustc_mir/borrow_check/mod.rs @@ -644,6 +644,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc } } StatementKind::Nop + | StatementKind::Coverage(..) | StatementKind::AscribeUserType(..) | StatementKind::Retag { .. } | StatementKind::StorageLive(..) => { diff --git a/src/librustc_mir/borrow_check/type_check/mod.rs b/src/librustc_mir/borrow_check/type_check/mod.rs index ff98de5475e..168a352591d 100644 --- a/src/librustc_mir/borrow_check/type_check/mod.rs +++ b/src/librustc_mir/borrow_check/type_check/mod.rs @@ -1532,6 +1532,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { | StatementKind::StorageDead(..) | StatementKind::LlvmInlineAsm { .. } | StatementKind::Retag { .. } + | StatementKind::Coverage(..) | StatementKind::Nop => {} } } diff --git a/src/librustc_mir/dataflow/impls/borrows.rs b/src/librustc_mir/dataflow/impls/borrows.rs index 7e7b7f2cc76..aeb7ffe3e3b 100644 --- a/src/librustc_mir/dataflow/impls/borrows.rs +++ b/src/librustc_mir/dataflow/impls/borrows.rs @@ -302,6 +302,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> { | mir::StatementKind::StorageLive(..) | mir::StatementKind::Retag { .. } | mir::StatementKind::AscribeUserType(..) + | mir::StatementKind::Coverage(..) | mir::StatementKind::Nop => {} } } diff --git a/src/librustc_mir/dataflow/impls/init_locals.rs b/src/librustc_mir/dataflow/impls/init_locals.rs index 01cb794a2e0..0e7cd1bb0e4 100644 --- a/src/librustc_mir/dataflow/impls/init_locals.rs +++ b/src/librustc_mir/dataflow/impls/init_locals.rs @@ -99,6 +99,7 @@ where PlaceContext::NonUse( NonUseContext::StorageLive | NonUseContext::AscribeUserTy + | NonUseContext::Coverage | NonUseContext::VarDebugInfo, ) | PlaceContext::NonMutatingUse( diff --git a/src/librustc_mir/dataflow/impls/storage_liveness.rs b/src/librustc_mir/dataflow/impls/storage_liveness.rs index cd04493c092..21623e3cad5 100644 --- a/src/librustc_mir/dataflow/impls/storage_liveness.rs +++ b/src/librustc_mir/dataflow/impls/storage_liveness.rs @@ -145,6 +145,7 @@ impl<'mir, 'tcx> dataflow::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, // Nothing to do for these. Match exhaustively so this fails to compile when new // variants are added. StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::FakeRead(..) | StatementKind::Nop | StatementKind::Retag(..) diff --git a/src/librustc_mir/dataflow/move_paths/builder.rs b/src/librustc_mir/dataflow/move_paths/builder.rs index 7c8aa1db71f..e567063e0d5 100644 --- a/src/librustc_mir/dataflow/move_paths/builder.rs +++ b/src/librustc_mir/dataflow/move_paths/builder.rs @@ -318,6 +318,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> { } StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => {} } } diff --git a/src/librustc_mir/interpret/intrinsics.rs b/src/librustc_mir/interpret/intrinsics.rs index 2c0a42a9bf3..b37dcd42f4c 100644 --- a/src/librustc_mir/interpret/intrinsics.rs +++ b/src/librustc_mir/interpret/intrinsics.rs @@ -440,11 +440,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // These just return their argument self.copy_op(args[0], dest)?; } - // FIXME(#73156): Handle source code coverage in const eval - sym::count_code_region - | sym::coverage_counter_add - | sym::coverage_counter_subtract - | sym::coverage_unreachable => (), _ => return Ok(false), } diff --git a/src/librustc_mir/interpret/step.rs b/src/librustc_mir/interpret/step.rs index adecee3f7cb..156da84f291 100644 --- a/src/librustc_mir/interpret/step.rs +++ b/src/librustc_mir/interpret/step.rs @@ -118,6 +118,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Statements we do not track. AscribeUserType(..) => {} + // Currently, Miri discards Coverage statements. Coverage statements are only injected + // via an optional compile time MIR pass and have no side effects. Since Coverage + // statements don't exist at the source level, it is safe for Miri to ignore them, even + // for undefined behavior (UB) checks. + // + // A coverage counter inside a const expression (for example, a counter injected in a + // const function) is discarded when the const is evaluated at compile time. Whether + // this should change, and/or how to implement a const eval counter, is a subject of the + // following issue: + // + // FIXME(#73156): Handle source code coverage in const eval + Coverage(..) => {} + // Defined to do nothing. These are added by optimization passes, to avoid changing the // size of MIR constantly. Nop => {} diff --git a/src/librustc_mir/transform/check_consts/validation.rs b/src/librustc_mir/transform/check_consts/validation.rs index 526c896df09..4812ef5a894 100644 --- a/src/librustc_mir/transform/check_consts/validation.rs +++ b/src/librustc_mir/transform/check_consts/validation.rs @@ -485,6 +485,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { | StatementKind::StorageDead(_) | StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => {} } } diff --git a/src/librustc_mir/transform/check_unsafety.rs b/src/librustc_mir/transform/check_unsafety.rs index 6aabc1941a6..c3e04e698db 100644 --- a/src/librustc_mir/transform/check_unsafety.rs +++ b/src/librustc_mir/transform/check_unsafety.rs @@ -114,6 +114,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { | StatementKind::StorageDead(..) | StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => { // safe (at least as emitted during MIR construction) } diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs index 8618cc126c5..52c5f4e25cf 100644 --- a/src/librustc_mir/transform/generator.rs +++ b/src/librustc_mir/transform/generator.rs @@ -1443,6 +1443,7 @@ impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { | StatementKind::StorageDead(_) | StatementKind::Retag(..) | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => {} } } diff --git a/src/librustc_mir/transform/instrument_coverage.rs b/src/librustc_mir/transform/instrument_coverage.rs index 500d66ece06..f60e6da714a 100644 --- a/src/librustc_mir/transform/instrument_coverage.rs +++ b/src/librustc_mir/transform/instrument_coverage.rs @@ -1,23 +1,16 @@ use crate::transform::{MirPass, MirSource}; -use crate::util::patch::MirPatch; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_hir::lang_items; use rustc_middle::hir; use rustc_middle::ich::StableHashingContext; use rustc_middle::mir; use rustc_middle::mir::coverage::*; -use rustc_middle::mir::interpret::Scalar; -use rustc_middle::mir::traversal; -use rustc_middle::mir::{ - BasicBlock, BasicBlockData, CoverageInfo, Operand, Place, SourceInfo, SourceScope, - StatementKind, Terminator, TerminatorKind, -}; -use rustc_middle::ty; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::{BasicBlock, Coverage, CoverageInfo, Location, Statement, StatementKind}; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{FnDef, TyCtxt}; +use rustc_middle::ty::TyCtxt; use rustc_span::def_id::DefId; -use rustc_span::{FileName, Pos, RealFileName, Span}; +use rustc_span::{FileName, Pos, RealFileName, Span, Symbol}; /// Inserts call to count_code_region() as a placeholder to be replaced during code generation with /// the intrinsic llvm.instrprof.increment. @@ -29,16 +22,29 @@ pub(crate) fn provide(providers: &mut Providers) { providers.coverageinfo = |tcx, def_id| coverageinfo_from_mir(tcx, def_id); } +struct CoverageVisitor { + info: CoverageInfo, +} + +impl Visitor<'_> for CoverageVisitor { + fn visit_coverage(&mut self, coverage: &Coverage, _location: Location) { + match coverage.kind { + CoverageKind::Counter { id, .. } => { + let counter_id = u32::from(id); + self.info.num_counters = std::cmp::max(self.info.num_counters, counter_id + 1); + } + CoverageKind::Expression { id, .. } => { + let expression_index = u32::MAX - u32::from(id); + self.info.num_expressions = + std::cmp::max(self.info.num_expressions, expression_index + 1); + } + _ => {} + } + } +} + fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, mir_def_id: DefId) -> CoverageInfo { let mir_body = tcx.optimized_mir(mir_def_id); - // FIXME(richkadel): The current implementation assumes the MIR for the given DefId - // represents a single function. Validate and/or correct if inlining (which should be disabled - // if -Zinstrument-coverage is enabled) and/or monomorphization invalidates these assumptions. - let count_code_region_fn = tcx.require_lang_item(lang_items::CountCodeRegionFnLangItem, None); - let coverage_counter_add_fn = - tcx.require_lang_item(lang_items::CoverageCounterAddFnLangItem, None); - let coverage_counter_subtract_fn = - tcx.require_lang_item(lang_items::CoverageCounterSubtractFnLangItem, None); // The `num_counters` argument to `llvm.instrprof.increment` is the number of injected // counters, with each counter having a counter ID from `0..num_counters-1`. MIR optimization @@ -49,49 +55,11 @@ fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, mir_def_id: DefId) -> Coverage // `num_expressions` is the number of counter expressions added to the MIR body. Both // `num_counters` and `num_expressions` are used to initialize new vectors, during backend // code generate, to lookup counters and expressions by simple u32 indexes. - let mut num_counters: u32 = 0; - let mut num_expressions: u32 = 0; - for terminator in - traversal::preorder(mir_body).map(|(_, data)| data).filter_map(call_terminators) - { - if let TerminatorKind::Call { func: Operand::Constant(func), args, .. } = &terminator.kind { - match func.literal.ty.kind { - FnDef(id, _) if id == count_code_region_fn => { - let counter_id_arg = - args.get(count_code_region_args::COUNTER_ID).expect("arg found"); - let counter_id = mir::Operand::scalar_from_const(counter_id_arg) - .to_u32() - .expect("counter_id arg is u32"); - num_counters = std::cmp::max(num_counters, counter_id + 1); - } - FnDef(id, _) - if id == coverage_counter_add_fn || id == coverage_counter_subtract_fn => - { - let expression_id_arg = args - .get(coverage_counter_expression_args::EXPRESSION_ID) - .expect("arg found"); - let id_descending_from_max = mir::Operand::scalar_from_const(expression_id_arg) - .to_u32() - .expect("expression_id arg is u32"); - // Counter expressions are initially assigned IDs descending from `u32::MAX`, so - // the range of expression IDs is disjoint from the range of counter IDs. This - // way, both counters and expressions can be operands in other expressions. - let expression_index = u32::MAX - id_descending_from_max; - num_expressions = std::cmp::max(num_expressions, expression_index + 1); - } - _ => {} - } - } - } - CoverageInfo { num_counters, num_expressions } -} + let mut coverage_visitor = + CoverageVisitor { info: CoverageInfo { num_counters: 0, num_expressions: 0 } }; -fn call_terminators(data: &'tcx BasicBlockData<'tcx>) -> Option<&'tcx Terminator<'tcx>> { - let terminator = data.terminator(); - match terminator.kind { - TerminatorKind::Call { .. } => Some(terminator), - _ => None, - } + coverage_visitor.visit_body(mir_body); + coverage_visitor.info } impl<'tcx> MirPass<'tcx> for InstrumentCoverage { @@ -104,19 +72,6 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage { } } -/// Distinguishes the expression operators. -enum Op { - Add, - Subtract, -} - -struct InjectedCall<'tcx> { - func: Operand<'tcx>, - args: Vec>, - span: Span, - inject_at: Span, -} - struct Instrumentor<'a, 'tcx> { tcx: TyCtxt<'tcx>, mir_def_id: DefId, @@ -143,21 +98,21 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } /// Counter IDs start from zero and go up. - fn next_counter(&mut self) -> u32 { + fn next_counter(&mut self) -> CounterValueReference { assert!(self.num_counters < u32::MAX - self.num_expressions); let next = self.num_counters; self.num_counters += 1; - next + CounterValueReference::from(next) } /// Expression IDs start from u32::MAX and go down because a CounterExpression can reference /// (add or subtract counts) of both Counter regions and CounterExpression regions. The counter /// expression operand IDs must be unique across both types. - fn next_expression(&mut self) -> u32 { + fn next_expression(&mut self) -> InjectedExpressionIndex { assert!(self.num_counters < u32::MAX - self.num_expressions); let next = u32::MAX - self.num_expressions; self.num_expressions += 1; - next + InjectedExpressionIndex::from(next) } fn function_source_hash(&mut self) -> u64 { @@ -172,24 +127,14 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } fn inject_counters(&mut self) { - let mir_body = &self.mir_body; let body_span = self.hir_body.value.span; debug!("instrumenting {:?}, span: {:?}", self.mir_def_id, body_span); // FIXME(richkadel): As a first step, counters are only injected at the top of each // function. The complete solution will inject counters at each conditional code branch. - let _ignore = mir_body; - let id = self.next_counter(); - let function_source_hash = self.function_source_hash(); - let scope = rustc_middle::mir::OUTERMOST_SOURCE_SCOPE; - let is_cleanup = false; - let next_block = rustc_middle::mir::START_BLOCK; - self.inject_call( - self.make_counter(id, function_source_hash, body_span), - scope, - is_cleanup, - next_block, - ); + let block = rustc_middle::mir::START_BLOCK; + let counter = self.make_counter(); + self.inject_statement(counter, body_span, block); // FIXME(richkadel): The next step to implement source based coverage analysis will be // instrumenting branches within functions, and some regions will be counted by "counter @@ -198,211 +143,86 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let fake_use = false; if fake_use { let add = false; - let lhs = 1; + let fake_counter = CoverageKind::Counter { + function_source_hash: self.function_source_hash(), + id: CounterValueReference::from_u32(1), + }; + let fake_expression = CoverageKind::Expression { + id: InjectedExpressionIndex::from(u32::MAX - 1), + lhs: ExpressionOperandId::from_u32(1), + op: Op::Add, + rhs: ExpressionOperandId::from_u32(2), + }; + + let lhs = fake_counter.as_operand_id(); let op = if add { Op::Add } else { Op::Subtract }; - let rhs = 2; + let rhs = fake_expression.as_operand_id(); - let scope = rustc_middle::mir::OUTERMOST_SOURCE_SCOPE; - let is_cleanup = false; - let next_block = rustc_middle::mir::START_BLOCK; + let block = rustc_middle::mir::START_BLOCK; - let id = self.next_expression(); - self.inject_call( - self.make_expression(id, body_span, lhs, op, rhs), - scope, - is_cleanup, - next_block, - ); + let expression = self.make_expression(lhs, op, rhs); + self.inject_statement(expression, body_span, block); } } - fn make_counter(&self, id: u32, function_source_hash: u64, span: Span) -> InjectedCall<'tcx> { - let inject_at = span.shrink_to_lo(); - - let func = function_handle( - self.tcx, - self.tcx.require_lang_item(lang_items::CountCodeRegionFnLangItem, None), - inject_at, - ); - - let mut args = Vec::new(); - - use count_code_region_args::*; - debug_assert_eq!(FUNCTION_SOURCE_HASH, args.len()); - args.push(self.const_u64(function_source_hash, inject_at)); - - debug_assert_eq!(COUNTER_ID, args.len()); - args.push(self.const_u32(id, inject_at)); - - InjectedCall { func, args, span, inject_at } + fn make_counter(&mut self) -> CoverageKind { + CoverageKind::Counter { + function_source_hash: self.function_source_hash(), + id: self.next_counter(), + } } fn make_expression( - &self, - id: u32, - span: Span, - lhs: u32, - op: Op, - rhs: u32, - ) -> InjectedCall<'tcx> { - let inject_at = span.shrink_to_lo(); - - let func = function_handle( - self.tcx, - self.tcx.require_lang_item( - match op { - Op::Add => lang_items::CoverageCounterAddFnLangItem, - Op::Subtract => lang_items::CoverageCounterSubtractFnLangItem, - }, - None, - ), - inject_at, - ); - - let mut args = Vec::new(); - - use coverage_counter_expression_args::*; - debug_assert_eq!(EXPRESSION_ID, args.len()); - args.push(self.const_u32(id, inject_at)); - - debug_assert_eq!(LEFT_ID, args.len()); - args.push(self.const_u32(lhs, inject_at)); - - debug_assert_eq!(RIGHT_ID, args.len()); - args.push(self.const_u32(rhs, inject_at)); - - InjectedCall { func, args, span, inject_at } - } - - fn inject_call( &mut self, - call: InjectedCall<'tcx>, - scope: SourceScope, - is_cleanup: bool, - next_block: BasicBlock, - ) { - let InjectedCall { func, mut args, span, inject_at } = call; - debug!( - " injecting {}call to {:?}({:?}) at: {:?}, scope: {:?}", - if is_cleanup { "cleanup " } else { "" }, - func, - args, - inject_at, - scope, - ); - - let mut patch = MirPatch::new(self.mir_body); - - let (file_name, start_line, start_col, end_line, end_col) = self.code_region(&span); - - // FIXME(richkadel): Note that `const_str()` results in the creation of an `Allocation` to - // hold one copy of each unique filename. It looks like that `Allocation` may translate into - // the creation of an `@alloc` in LLVM IR that is never actually used by runtime code. - // - // Example LLVM IR: - // - // @alloc4 = private unnamed_addr constant <{ [43 x i8] }> \ - // <{ [43 x i8] c"C:\\msys64\\home\\richkadel\\rust\\rust_basic.rs" }>, align 1 - // - // Can I flag the alloc as something not to be added to codegen? Or somehow remove it before - // it gets added to the LLVM IR? Do we need some kind of reference counting to know it's - // not used by any runtime code? - // - // This question is moot if I convert the Call Terminators to Statements, I believe: - // https://rust-lang.zulipchat.com/#narrow/stream/233931-t-compiler.2Fmajor-changes/topic/Implement.20LLVM-compatible.20source-based.20cod.20compiler-team.23278/near/206731748 - args.push(self.const_str(&file_name, inject_at)); - args.push(self.const_u32(start_line, inject_at)); - args.push(self.const_u32(start_col, inject_at)); - args.push(self.const_u32(end_line, inject_at)); - args.push(self.const_u32(end_col, inject_at)); - - let temp = patch.new_temp(self.tcx.mk_unit(), inject_at); - let new_block = patch.new_block(placeholder_block(inject_at, scope, is_cleanup)); - patch.patch_terminator( - new_block, - TerminatorKind::Call { - func, - args, - // new_block will swapped with the next_block, after applying patch - destination: Some((Place::from(temp), new_block)), - cleanup: None, - from_hir_call: false, - fn_span: inject_at, - }, - ); - - patch.add_statement(new_block.start_location(), StatementKind::StorageLive(temp)); - patch.add_statement(next_block.start_location(), StatementKind::StorageDead(temp)); - - patch.apply(self.mir_body); - - // To insert the `new_block` in front of the first block in the counted branch (the - // `next_block`), just swap the indexes, leaving the rest of the graph unchanged. - self.mir_body.basic_blocks_mut().swap(next_block, new_block); + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + ) -> CoverageKind { + CoverageKind::Expression { id: self.next_expression(), lhs, op, rhs } } - /// Convert the Span into its file name, start line and column, and end line and column - fn code_region(&self, span: &Span) -> (String, u32, u32, u32, u32) { - let source_map = self.tcx.sess.source_map(); - let start = source_map.lookup_char_pos(span.lo()); - let end = if span.hi() == span.lo() { - start.clone() - } else { - let end = source_map.lookup_char_pos(span.hi()); - debug_assert_eq!( - start.file.name, - end.file.name, - "Region start ({:?} -> {:?}) and end ({:?} -> {:?}) don't come from the same source file!", - span.lo(), - start, - span.hi(), - end - ); - end + fn inject_statement(&mut self, coverage_kind: CoverageKind, span: Span, block: BasicBlock) { + let code_region = make_code_region(self.tcx, &span); + debug!(" injecting statement {:?} covering {:?}", coverage_kind, code_region); + + let data = &mut self.mir_body[block]; + let source_info = data.terminator().source_info; + let statement = Statement { + source_info, + kind: StatementKind::Coverage(box Coverage { kind: coverage_kind, code_region }), }; - match &start.file.name { - FileName::Real(RealFileName::Named(path)) => ( - path.to_string_lossy().to_string(), - start.line as u32, - start.col.to_u32() + 1, - end.line as u32, - end.col.to_u32() + 1, - ), - _ => { - bug!("start.file.name should be a RealFileName, but it was: {:?}", start.file.name) - } - } - } - - fn const_str(&self, value: &str, span: Span) -> Operand<'tcx> { - Operand::const_from_str(self.tcx, value, span) - } - - fn const_u32(&self, value: u32, span: Span) -> Operand<'tcx> { - Operand::const_from_scalar(self.tcx, self.tcx.types.u32, Scalar::from_u32(value), span) - } - - fn const_u64(&self, value: u64, span: Span) -> Operand<'tcx> { - Operand::const_from_scalar(self.tcx, self.tcx.types.u64, Scalar::from_u64(value), span) + data.statements.push(statement); } } -fn function_handle<'tcx>(tcx: TyCtxt<'tcx>, fn_def_id: DefId, span: Span) -> Operand<'tcx> { - let ret_ty = tcx.fn_sig(fn_def_id).output(); - let ret_ty = ret_ty.no_bound_vars().unwrap(); - let substs = tcx.mk_substs(::std::iter::once(ty::subst::GenericArg::from(ret_ty))); - Operand::function_handle(tcx, fn_def_id, substs, span) -} - -fn placeholder_block(span: Span, scope: SourceScope, is_cleanup: bool) -> BasicBlockData<'tcx> { - BasicBlockData { - statements: vec![], - terminator: Some(Terminator { - source_info: SourceInfo { span, scope }, - // this gets overwritten by the counter Call - kind: TerminatorKind::Unreachable, - }), - is_cleanup, +/// Convert the Span into its file name, start line and column, and end line and column +fn make_code_region<'tcx>(tcx: TyCtxt<'tcx>, span: &Span) -> CodeRegion { + let source_map = tcx.sess.source_map(); + let start = source_map.lookup_char_pos(span.lo()); + let end = if span.hi() == span.lo() { + start.clone() + } else { + let end = source_map.lookup_char_pos(span.hi()); + debug_assert_eq!( + start.file.name, + end.file.name, + "Region start ({:?} -> {:?}) and end ({:?} -> {:?}) don't come from the same source file!", + span.lo(), + start, + span.hi(), + end + ); + end + }; + match &start.file.name { + FileName::Real(RealFileName::Named(path)) => CodeRegion { + file_name: Symbol::intern(&path.to_string_lossy()), + start_line: start.line as u32, + start_col: start.col.to_u32() + 1, + end_line: end.line as u32, + end_col: end.col.to_u32() + 1, + }, + _ => bug!("start.file.name should be a RealFileName, but it was: {:?}", start.file.name), } } diff --git a/src/librustc_mir/transform/qualify_min_const_fn.rs b/src/librustc_mir/transform/qualify_min_const_fn.rs index 7abb09885ff..26db4600a2b 100644 --- a/src/librustc_mir/transform/qualify_min_const_fn.rs +++ b/src/librustc_mir/transform/qualify_min_const_fn.rs @@ -273,6 +273,7 @@ fn check_statement( | StatementKind::StorageDead(_) | StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => Ok(()), } } diff --git a/src/librustc_mir/transform/remove_noop_landing_pads.rs b/src/librustc_mir/transform/remove_noop_landing_pads.rs index bf63bf24447..0bad1e5037a 100644 --- a/src/librustc_mir/transform/remove_noop_landing_pads.rs +++ b/src/librustc_mir/transform/remove_noop_landing_pads.rs @@ -38,6 +38,7 @@ impl RemoveNoopLandingPads { | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) | StatementKind::Nop => { // These are all nops in a landing pad } diff --git a/src/librustc_passes/weak_lang_items.rs b/src/librustc_passes/weak_lang_items.rs index 2749b96bc85..d5ce82ae28f 100644 --- a/src/librustc_passes/weak_lang_items.rs +++ b/src/librustc_passes/weak_lang_items.rs @@ -3,7 +3,6 @@ use rustc_data_structures::fx::FxHashSet; use rustc_errors::struct_span_err; use rustc_hir as hir; -use rustc_hir::fake_lang_items::FAKE_ITEMS_REFS; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::lang_items; use rustc_hir::weak_lang_items::WEAK_ITEMS_REFS; @@ -71,20 +70,11 @@ fn verify<'tcx>(tcx: TyCtxt<'tcx>, items: &lang_items::LanguageItems) { } impl<'a, 'tcx> Context<'a, 'tcx> { - fn register(&mut self, name: Symbol, span: Span, hir_id: hir::HirId) { + fn register(&mut self, name: Symbol, span: Span) { if let Some(&item) = WEAK_ITEMS_REFS.get(&name) { if self.items.require(item).is_err() { self.items.missing.push(item); } - } else if let Some(&item) = FAKE_ITEMS_REFS.get(&name) { - // Ensure "fake lang items" are registered. These are `extern` lang items that are - // injected into the MIR automatically (such as source code coverage counters), but are - // never actually linked; therefore, unlike "weak lang items", they cannot by registered - // when used, because they never appear to be used. - if self.items.items[item as usize].is_none() { - let item_def_id = self.tcx.hir().local_def_id(hir_id).to_def_id(); - self.items.items[item as usize] = Some(item_def_id); - } } else { struct_span_err!(self.tcx.sess, span, E0264, "unknown external lang item: `{}`", name) .emit(); @@ -102,7 +92,7 @@ impl<'a, 'tcx, 'v> Visitor<'v> for Context<'a, 'tcx> { fn visit_foreign_item(&mut self, i: &hir::ForeignItem<'_>) { let check_name = |attr, sym| self.tcx.sess.check_name(attr, sym); if let Some((lang_item, _)) = hir::lang_items::extract(check_name, &i.attrs) { - self.register(lang_item, i.span, i.hir_id); + self.register(lang_item, i.span); } intravisit::walk_foreign_item(self, i) } diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs index bc7efd26f46..3883d86520f 100644 --- a/src/librustc_span/symbol.rs +++ b/src/librustc_span/symbol.rs @@ -379,10 +379,6 @@ symbols! { core_intrinsics, cosf32, cosf64, - count_code_region, - coverage_counter_add, - coverage_counter_subtract, - coverage_unreachable, crate_id, crate_in_paths, crate_local, diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index b3287caa0bf..47cea8649ef 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -379,47 +379,6 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { sym::nontemporal_store => (1, vec![tcx.mk_mut_ptr(param(0)), param(0)], tcx.mk_unit()), - sym::count_code_region => ( - 0, - vec![ - tcx.types.u64, - tcx.types.u32, - tcx.mk_static_str(), - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - ], - tcx.mk_unit(), - ), - - sym::coverage_counter_add | sym::coverage_counter_subtract => ( - 0, - vec![ - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - tcx.mk_static_str(), - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - ], - tcx.mk_unit(), - ), - - sym::coverage_unreachable => ( - 0, - vec![ - tcx.mk_static_str(), - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - tcx.types.u32, - ], - tcx.mk_unit(), - ), - other => { struct_span_err!( tcx.sess, diff --git a/src/test/mir-opt/instrument_coverage.bar.InstrumentCoverage.diff b/src/test/mir-opt/instrument_coverage.bar.InstrumentCoverage.diff index f048757ad86..01d837d155a 100644 --- a/src/test/mir-opt/instrument_coverage.bar.InstrumentCoverage.diff +++ b/src/test/mir-opt/instrument_coverage.bar.InstrumentCoverage.diff @@ -3,32 +3,10 @@ fn bar() -> bool { let mut _0: bool; // return place in scope 0 at /the/src/instrument_coverage.rs:19:13: 19:17 -+ let mut _1: (); // in scope 0 at /the/src/instrument_coverage.rs:19:18: 19:18 bb0: { -+ StorageLive(_1); // scope 0 at /the/src/instrument_coverage.rs:19:18: 19:18 -+ _1 = const std::intrinsics::count_code_region(const 10208505205182607101_u64, const 0_u32, const "/the/src/instrument_coverage.rs", const 19_u32, const 18_u32, const 21_u32, const 2_u32) -> bb2; // scope 0 at /the/src/instrument_coverage.rs:19:18: 19:18 -+ // ty::Const -+ // + ty: unsafe extern "rust-intrinsic" fn(u64, u32, &'static str, u32, u32, u32, u32) {std::intrinsics::count_code_region} -+ // + val: Value(Scalar()) -+ // mir::Constant -+ // + span: /the/src/instrument_coverage.rs:19:18: 19:18 -+ // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u64, u32, &'static str, u32, u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar()) } -+ // ty::Const -+ // + ty: &str -+ // + val: Value(Slice { data: Allocation { bytes: [47, 116, 104, 101, 47, 115, 114, 99, 47, 105, 110, 115, 116, 114, 117, 109, 101, 110, 116, 95, 99, 111, 118, 101, 114, 97, 103, 101, 46, 114, 115], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [2147483647], len: Size { raw: 31 } }, size: Size { raw: 31 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 31 }) -+ // mir::Constant -+ // + span: /the/src/instrument_coverage.rs:19:18: 19:18 -+ // + literal: Const { ty: &str, val: Value(Slice { data: Allocation { bytes: [47, 116, 104, 101, 47, 115, 114, 99, 47, 105, 110, 115, 116, 114, 117, 109, 101, 110, 116, 95, 99, 111, 118, 101, 114, 97, 103, 101, 46, 114, 115], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [2147483647], len: Size { raw: 31 } }, size: Size { raw: 31 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 31 }) } -+ } -+ -+ bb1 (cleanup): { -+ resume; // scope 0 at /the/src/instrument_coverage.rs:19:1: 21:2 -+ } -+ -+ bb2: { -+ StorageDead(_1); // scope 0 at /the/src/instrument_coverage.rs:20:5: 20:9 _0 = const true; // scope 0 at /the/src/instrument_coverage.rs:20:5: 20:9 ++ Coverage { kind: Counter { function_source_hash: 10208505205182607101, id: CounterValueReference(0) }, code_region: /the/src/instrument_coverage.rs:19:18 - 21:2 }; // scope 0 at /the/src/instrument_coverage.rs:21:2: 21:2 return; // scope 0 at /the/src/instrument_coverage.rs:21:2: 21:2 } } diff --git a/src/test/mir-opt/instrument_coverage.main.InstrumentCoverage.diff b/src/test/mir-opt/instrument_coverage.main.InstrumentCoverage.diff index 931ebf5e140..c3831cb2bb4 100644 --- a/src/test/mir-opt/instrument_coverage.main.InstrumentCoverage.diff +++ b/src/test/mir-opt/instrument_coverage.main.InstrumentCoverage.diff @@ -6,24 +6,10 @@ let mut _1: (); // in scope 0 at /the/src/instrument_coverage.rs:10:1: 16:2 let mut _2: bool; // in scope 0 at /the/src/instrument_coverage.rs:12:12: 12:17 let mut _3: !; // in scope 0 at /the/src/instrument_coverage.rs:12:18: 14:10 -+ let mut _4: (); // in scope 0 at /the/src/instrument_coverage.rs:10:11: 10:11 bb0: { -- falseUnwind -> [real: bb1, cleanup: bb2]; // scope 0 at /the/src/instrument_coverage.rs:11:5: 15:6 -+ StorageLive(_4); // scope 0 at /the/src/instrument_coverage.rs:10:11: 10:11 -+ _4 = const std::intrinsics::count_code_region(const 16004455475339839479_u64, const 0_u32, const "/the/src/instrument_coverage.rs", const 10_u32, const 11_u32, const 16_u32, const 2_u32) -> bb7; // scope 0 at /the/src/instrument_coverage.rs:10:11: 10:11 -+ // ty::Const -+ // + ty: unsafe extern "rust-intrinsic" fn(u64, u32, &'static str, u32, u32, u32, u32) {std::intrinsics::count_code_region} -+ // + val: Value(Scalar()) -+ // mir::Constant -+ // + span: /the/src/instrument_coverage.rs:10:11: 10:11 -+ // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u64, u32, &'static str, u32, u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar()) } -+ // ty::Const -+ // + ty: &str -+ // + val: Value(Slice { data: Allocation { bytes: [47, 116, 104, 101, 47, 115, 114, 99, 47, 105, 110, 115, 116, 114, 117, 109, 101, 110, 116, 95, 99, 111, 118, 101, 114, 97, 103, 101, 46, 114, 115], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [2147483647], len: Size { raw: 31 } }, size: Size { raw: 31 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 31 }) -+ // mir::Constant -+ // + span: /the/src/instrument_coverage.rs:10:11: 10:11 -+ // + literal: Const { ty: &str, val: Value(Slice { data: Allocation { bytes: [47, 116, 104, 101, 47, 115, 114, 99, 47, 105, 110, 115, 116, 114, 117, 109, 101, 110, 116, 95, 99, 111, 118, 101, 114, 97, 103, 101, 46, 114, 115], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [2147483647], len: Size { raw: 31 } }, size: Size { raw: 31 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 31 }) } ++ Coverage { kind: Counter { function_source_hash: 16004455475339839479, id: CounterValueReference(0) }, code_region: /the/src/instrument_coverage.rs:10:11 - 16:2 }; // scope 0 at /the/src/instrument_coverage.rs:11:5: 15:6 + falseUnwind -> [real: bb1, cleanup: bb2]; // scope 0 at /the/src/instrument_coverage.rs:11:5: 15:6 } bb1: { @@ -72,11 +58,6 @@ // + literal: Const { ty: (), val: Value(Scalar()) } StorageDead(_2); // scope 0 at /the/src/instrument_coverage.rs:15:5: 15:6 return; // scope 0 at /the/src/instrument_coverage.rs:16:2: 16:2 -+ } -+ -+ bb7: { -+ StorageDead(_4); // scope 0 at /the/src/instrument_coverage.rs:11:5: 15:6 -+ falseUnwind -> [real: bb1, cleanup: bb2]; // scope 0 at /the/src/instrument_coverage.rs:11:5: 15:6 } } diff --git a/src/test/mir-opt/instrument_coverage.rs b/src/test/mir-opt/instrument_coverage.rs index 430573c7c05..d9c8e8d43a4 100644 --- a/src/test/mir-opt/instrument_coverage.rs +++ b/src/test/mir-opt/instrument_coverage.rs @@ -1,5 +1,5 @@ -// Test that the initial version of Rust coverage injects count_code_region() placeholder calls, -// at the top of each function. The placeholders are later converted into LLVM instrprof.increment +// Test that the initial version of Rust coverage injects Coverage statements at the top of each +// function. The Coverage Counter statements are later converted into LLVM instrprof.increment // intrinsics, during codegen. // needs-profiler-support