Auto merge of #79049 - tmiasko:lower-intrinsics, r=jonas-schievink

Lower intrinsics calls: forget, size_of, unreachable, wrapping_*

This allows constant propagation to evaluate `size_of` and `wrapping_*`,
and unreachable propagation to propagate a call to `unreachable`.

The lowering is performed as a MIR optimization, rather than during MIR
building to preserve the special status of intrinsics with respect to
unsafety checks and promotion.

Currently enabled by default to determine the performance impact (no
significant impact expected). In practice only useful when combined with
inlining since intrinsics are rarely used directly (with exception of
`unreachable` and `discriminant_value` used by built-in derive macros).

Closes #32716.
This commit is contained in:
bors 2020-11-14 22:05:54 +00:00
commit 361c4ea224
10 changed files with 373 additions and 5 deletions

View File

@ -0,0 +1,108 @@
//! Lowers intrinsic calls
use crate::transform::MirPass;
use rustc_middle::mir::*;
use rustc_middle::ty::subst::SubstsRef;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::symbol::{sym, Symbol};
use rustc_target::spec::abi::Abi;
pub struct LowerIntrinsics;
impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
for block in body.basic_blocks_mut() {
let terminator = block.terminator.as_mut().unwrap();
if let TerminatorKind::Call {
func: Operand::Constant(box Constant { literal: ty::Const { ty: func_ty, .. }, .. }),
args,
destination,
..
} = &mut terminator.kind
{
let (intrinsic_name, substs) = match resolve_rust_intrinsic(tcx, func_ty) {
None => continue,
Some(it) => it,
};
match intrinsic_name {
sym::unreachable => {
terminator.kind = TerminatorKind::Unreachable;
}
sym::forget => {
if let Some((destination, target)) = *destination {
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::Assign(box (
destination,
Rvalue::Use(Operand::Constant(box Constant {
span: terminator.source_info.span,
user_ty: None,
literal: ty::Const::zero_sized(tcx, tcx.types.unit),
})),
)),
});
terminator.kind = TerminatorKind::Goto { target };
}
}
sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
if let Some((destination, target)) = *destination {
let lhs;
let rhs;
{
let mut args = args.drain(..);
lhs = args.next().unwrap();
rhs = args.next().unwrap();
}
let bin_op = match intrinsic_name {
sym::wrapping_add => BinOp::Add,
sym::wrapping_sub => BinOp::Sub,
sym::wrapping_mul => BinOp::Mul,
_ => bug!("unexpected intrinsic"),
};
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::Assign(box (
destination,
Rvalue::BinaryOp(bin_op, lhs, rhs),
)),
});
terminator.kind = TerminatorKind::Goto { target };
}
}
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
// The checked binary operations are not suitable target for lowering here,
// since their semantics depend on the value of overflow-checks flag used
// during codegen. Issue #35310.
}
sym::size_of => {
if let Some((destination, target)) = *destination {
let tp_ty = substs.type_at(0);
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::Assign(box (
destination,
Rvalue::NullaryOp(NullOp::SizeOf, tp_ty),
)),
});
terminator.kind = TerminatorKind::Goto { target };
}
}
_ => {}
}
}
}
}
}
fn resolve_rust_intrinsic(
tcx: TyCtxt<'tcx>,
func_ty: Ty<'tcx>,
) -> Option<(Symbol, SubstsRef<'tcx>)> {
if let ty::FnDef(def_id, substs) = *func_ty.kind() {
let fn_sig = func_ty.fn_sig(tcx);
if fn_sig.abi() == Abi::RustIntrinsic {
return Some((tcx.item_name(def_id), substs));
}
}
None
}

View File

@ -32,6 +32,7 @@ pub mod function_item_references;
pub mod generator;
pub mod inline;
pub mod instcombine;
pub mod lower_intrinsics;
pub mod match_branches;
pub mod multiple_return_terminators;
pub mod no_landing_pads;
@ -390,6 +391,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
// The main optimizations that we do on MIR.
let optimizations: &[&dyn MirPass<'tcx>] = &[
&lower_intrinsics::LowerIntrinsics,
&remove_unneeded_drops::RemoveUnneededDrops,
&match_branches::MatchBranchSimplification,
// inst combine is after MatchBranchSimplification to clean up Ne(_1, false)

View File

@ -0,0 +1,27 @@
// MIR for `f_u64` before PreCodegen
fn f_u64() -> () {
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:34:16: 34:16
scope 1 (inlined f_dispatch::<u64>) { // at $DIR/lower_intrinsics.rs:35:5: 35:21
debug t => _2; // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
let _1: (); // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
let mut _2: u64; // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
scope 2 (inlined std::mem::size_of::<u64>) { // at $DIR/lower_intrinsics.rs:35:5: 35:21
}
}
bb0: {
_2 = const 0_u64; // scope 0 at $DIR/lower_intrinsics.rs:35:5: 35:21
StorageLive(_1); // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
_1 = f_non_zst::<u64>(move _2) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
// mir::Constant
// + span: $DIR/lower_intrinsics.rs:35:5: 35:21
// + literal: Const { ty: fn(u64) {f_non_zst::<u64>}, val: Value(Scalar(<ZST>)) }
}
bb1: {
StorageDead(_1); // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:34:16: 36:2
return; // scope 0 at $DIR/lower_intrinsics.rs:36:2: 36:2
}
}

View File

@ -0,0 +1,28 @@
// MIR for `f_unit` before PreCodegen
fn f_unit() -> () {
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:28:17: 28:17
let mut _1: (); // in scope 0 at $DIR/lower_intrinsics.rs:29:16: 29:18
scope 1 (inlined f_dispatch::<()>) { // at $DIR/lower_intrinsics.rs:29:5: 29:19
debug t => _1; // in scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
let _2: (); // in scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
scope 2 (inlined std::mem::size_of::<()>) { // at $DIR/lower_intrinsics.rs:29:5: 29:19
}
}
bb0: {
StorageLive(_1); // scope 0 at $DIR/lower_intrinsics.rs:29:16: 29:18
StorageLive(_2); // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
_2 = f_zst::<()>(const ()) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
// mir::Constant
// + span: $DIR/lower_intrinsics.rs:29:5: 29:19
// + literal: Const { ty: fn(()) {f_zst::<()>}, val: Value(Scalar(<ZST>)) }
}
bb1: {
StorageDead(_2); // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
StorageDead(_1); // scope 0 at $DIR/lower_intrinsics.rs:29:18: 29:19
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:28:17: 30:2
return; // scope 0 at $DIR/lower_intrinsics.rs:30:2: 30:2
}
}

View File

@ -0,0 +1,31 @@
- // MIR for `forget` before LowerIntrinsics
+ // MIR for `forget` after LowerIntrinsics
fn forget(_1: T) -> () {
debug t => _1; // in scope 0 at $DIR/lower_intrinsics.rs:18:18: 18:19
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:18:24: 18:24
let _2: (); // in scope 0 at $DIR/lower_intrinsics.rs:19:14: 19:41
let mut _3: T; // in scope 0 at $DIR/lower_intrinsics.rs:19:39: 19:40
scope 1 {
}
bb0: {
StorageLive(_2); // scope 0 at $DIR/lower_intrinsics.rs:19:5: 19:43
StorageLive(_3); // scope 1 at $DIR/lower_intrinsics.rs:19:39: 19:40
_3 = move _1; // scope 1 at $DIR/lower_intrinsics.rs:19:39: 19:40
- _2 = std::intrinsics::forget::<T>(move _3) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:19:14: 19:38
- // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(T) {std::intrinsics::forget::<T>}, val: Value(Scalar(<ZST>)) }
+ _2 = const (); // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
+ goto -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
}
bb1: {
StorageDead(_3); // scope 1 at $DIR/lower_intrinsics.rs:19:40: 19:41
StorageDead(_2); // scope 0 at $DIR/lower_intrinsics.rs:19:43: 19:44
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:18:24: 20:2
return; // scope 0 at $DIR/lower_intrinsics.rs:20:2: 20:2
}
}

View File

@ -0,0 +1,52 @@
// compile-flags: -Cpanic=abort
#![feature(core_intrinsics)]
#![crate_type = "lib"]
// EMIT_MIR lower_intrinsics.wrapping.LowerIntrinsics.diff
pub fn wrapping<T: Copy>(a: T, b: T) {
let _x = core::intrinsics::wrapping_add(a, b);
let _y = core::intrinsics::wrapping_sub(a, b);
let _z = core::intrinsics::wrapping_mul(a, b);
}
// EMIT_MIR lower_intrinsics.size_of.LowerIntrinsics.diff
pub fn size_of<T>() -> usize {
core::intrinsics::size_of::<T>()
}
// EMIT_MIR lower_intrinsics.forget.LowerIntrinsics.diff
pub fn forget<T>(t: T) {
unsafe { core::intrinsics::forget(t) };
}
// EMIT_MIR lower_intrinsics.unreachable.LowerIntrinsics.diff
pub fn unreachable() -> ! {
unsafe { core::intrinsics::unreachable() };
}
// EMIT_MIR lower_intrinsics.f_unit.PreCodegen.before.mir
pub fn f_unit() {
f_dispatch(());
}
// EMIT_MIR lower_intrinsics.f_u64.PreCodegen.before.mir
pub fn f_u64() {
f_dispatch(0u64);
}
#[inline(always)]
pub fn f_dispatch<T>(t: T) {
if std::mem::size_of::<T>() == 0 {
f_zst(t);
} else {
f_non_zst(t);
}
}
#[inline(never)]
pub fn f_zst<T>(t: T) {
}
#[inline(never)]
pub fn f_non_zst<T>(t: T) {}

View File

@ -0,0 +1,20 @@
- // MIR for `size_of` before LowerIntrinsics
+ // MIR for `size_of` after LowerIntrinsics
fn size_of() -> usize {
let mut _0: usize; // return place in scope 0 at $DIR/lower_intrinsics.rs:13:24: 13:29
bb0: {
- _0 = std::intrinsics::size_of::<T>() -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:14:5: 14:35
- // + literal: Const { ty: extern "rust-intrinsic" fn() -> usize {std::intrinsics::size_of::<T>}, val: Value(Scalar(<ZST>)) }
+ _0 = SizeOf(T); // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
+ goto -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
}
bb1: {
return; // scope 0 at $DIR/lower_intrinsics.rs:15:2: 15:2
}
}

View File

@ -0,0 +1,22 @@
- // MIR for `unreachable` before LowerIntrinsics
+ // MIR for `unreachable` after LowerIntrinsics
fn unreachable() -> ! {
let mut _0: !; // return place in scope 0 at $DIR/lower_intrinsics.rs:23:25: 23:26
let mut _1: !; // in scope 0 at $DIR/lower_intrinsics.rs:23:27: 25:2
let _2: (); // in scope 0 at $DIR/lower_intrinsics.rs:24:14: 24:45
let mut _3: !; // in scope 0 at $DIR/lower_intrinsics.rs:24:14: 24:45
scope 1 {
}
bb0: {
StorageLive(_2); // scope 0 at $DIR/lower_intrinsics.rs:24:5: 24:47
StorageLive(_3); // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
- std::intrinsics::unreachable(); // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:24:14: 24:43
- // + literal: Const { ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}, val: Value(Scalar(<ZST>)) }
+ unreachable; // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
}
}

View File

@ -0,0 +1,83 @@
- // MIR for `wrapping` before LowerIntrinsics
+ // MIR for `wrapping` after LowerIntrinsics
fn wrapping(_1: T, _2: T) -> () {
debug a => _1; // in scope 0 at $DIR/lower_intrinsics.rs:6:26: 6:27
debug b => _2; // in scope 0 at $DIR/lower_intrinsics.rs:6:32: 6:33
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:6:38: 6:38
let _3: T; // in scope 0 at $DIR/lower_intrinsics.rs:7:9: 7:11
let mut _4: T; // in scope 0 at $DIR/lower_intrinsics.rs:7:45: 7:46
let mut _5: T; // in scope 0 at $DIR/lower_intrinsics.rs:7:48: 7:49
let mut _7: T; // in scope 0 at $DIR/lower_intrinsics.rs:8:45: 8:46
let mut _8: T; // in scope 0 at $DIR/lower_intrinsics.rs:8:48: 8:49
let mut _10: T; // in scope 0 at $DIR/lower_intrinsics.rs:9:45: 9:46
let mut _11: T; // in scope 0 at $DIR/lower_intrinsics.rs:9:48: 9:49
scope 1 {
debug _x => _3; // in scope 1 at $DIR/lower_intrinsics.rs:7:9: 7:11
let _6: T; // in scope 1 at $DIR/lower_intrinsics.rs:8:9: 8:11
scope 2 {
debug _y => _6; // in scope 2 at $DIR/lower_intrinsics.rs:8:9: 8:11
let _9: T; // in scope 2 at $DIR/lower_intrinsics.rs:9:9: 9:11
scope 3 {
debug _z => _9; // in scope 3 at $DIR/lower_intrinsics.rs:9:9: 9:11
}
}
}
bb0: {
StorageLive(_3); // scope 0 at $DIR/lower_intrinsics.rs:7:9: 7:11
StorageLive(_4); // scope 0 at $DIR/lower_intrinsics.rs:7:45: 7:46
_4 = _1; // scope 0 at $DIR/lower_intrinsics.rs:7:45: 7:46
StorageLive(_5); // scope 0 at $DIR/lower_intrinsics.rs:7:48: 7:49
_5 = _2; // scope 0 at $DIR/lower_intrinsics.rs:7:48: 7:49
- _3 = wrapping_add::<T>(move _4, move _5) -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:7:14: 7:50
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:7:14: 7:44
- // + literal: Const { ty: extern "rust-intrinsic" fn(T, T) -> T {std::intrinsics::wrapping_add::<T>}, val: Value(Scalar(<ZST>)) }
+ _3 = Add(move _4, move _5); // scope 0 at $DIR/lower_intrinsics.rs:7:14: 7:50
+ goto -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:7:14: 7:50
}
bb1: {
StorageDead(_5); // scope 0 at $DIR/lower_intrinsics.rs:7:49: 7:50
StorageDead(_4); // scope 0 at $DIR/lower_intrinsics.rs:7:49: 7:50
StorageLive(_6); // scope 1 at $DIR/lower_intrinsics.rs:8:9: 8:11
StorageLive(_7); // scope 1 at $DIR/lower_intrinsics.rs:8:45: 8:46
_7 = _1; // scope 1 at $DIR/lower_intrinsics.rs:8:45: 8:46
StorageLive(_8); // scope 1 at $DIR/lower_intrinsics.rs:8:48: 8:49
_8 = _2; // scope 1 at $DIR/lower_intrinsics.rs:8:48: 8:49
- _6 = wrapping_sub::<T>(move _7, move _8) -> bb2; // scope 1 at $DIR/lower_intrinsics.rs:8:14: 8:50
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:8:14: 8:44
- // + literal: Const { ty: extern "rust-intrinsic" fn(T, T) -> T {std::intrinsics::wrapping_sub::<T>}, val: Value(Scalar(<ZST>)) }
+ _6 = Sub(move _7, move _8); // scope 1 at $DIR/lower_intrinsics.rs:8:14: 8:50
+ goto -> bb2; // scope 1 at $DIR/lower_intrinsics.rs:8:14: 8:50
}
bb2: {
StorageDead(_8); // scope 1 at $DIR/lower_intrinsics.rs:8:49: 8:50
StorageDead(_7); // scope 1 at $DIR/lower_intrinsics.rs:8:49: 8:50
StorageLive(_9); // scope 2 at $DIR/lower_intrinsics.rs:9:9: 9:11
StorageLive(_10); // scope 2 at $DIR/lower_intrinsics.rs:9:45: 9:46
_10 = _1; // scope 2 at $DIR/lower_intrinsics.rs:9:45: 9:46
StorageLive(_11); // scope 2 at $DIR/lower_intrinsics.rs:9:48: 9:49
_11 = _2; // scope 2 at $DIR/lower_intrinsics.rs:9:48: 9:49
- _9 = wrapping_mul::<T>(move _10, move _11) -> bb3; // scope 2 at $DIR/lower_intrinsics.rs:9:14: 9:50
- // mir::Constant
- // + span: $DIR/lower_intrinsics.rs:9:14: 9:44
- // + literal: Const { ty: extern "rust-intrinsic" fn(T, T) -> T {std::intrinsics::wrapping_mul::<T>}, val: Value(Scalar(<ZST>)) }
+ _9 = Mul(move _10, move _11); // scope 2 at $DIR/lower_intrinsics.rs:9:14: 9:50
+ goto -> bb3; // scope 2 at $DIR/lower_intrinsics.rs:9:14: 9:50
}
bb3: {
StorageDead(_11); // scope 2 at $DIR/lower_intrinsics.rs:9:49: 9:50
StorageDead(_10); // scope 2 at $DIR/lower_intrinsics.rs:9:49: 9:50
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:6:38: 10:2
StorageDead(_9); // scope 2 at $DIR/lower_intrinsics.rs:10:1: 10:2
StorageDead(_6); // scope 1 at $DIR/lower_intrinsics.rs:10:1: 10:2
StorageDead(_3); // scope 0 at $DIR/lower_intrinsics.rs:10:1: 10:2
return; // scope 0 at $DIR/lower_intrinsics.rs:10:2: 10:2
}
}

View File

@ -19,11 +19,6 @@ note: ...which requires const-evaluating + checking `std::mem::size_of`...
|
LL | pub const fn size_of<T>() -> usize {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
note: ...which requires simplifying constant for the type system `std::intrinsics::size_of`...
--> $SRC_DIR/core/src/intrinsics.rs:LL:COL
|
LL | pub fn size_of<T>() -> usize;
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
= note: ...which requires computing layout of `Foo`...
= note: ...which requires normalizing `[u8; _]`...
= note: ...which again requires simplifying constant for the type system `Foo::bytes::{constant#0}`, completing the cycle