OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rollup merge of #77511 - JulianKnodt:st_kind_cpy, r=oli-obk 

Add StatementKind::CopyNonOverlapping

Implements https://github.com/rust-lang/compiler-team/issues/348

r? `@nagisa`
This commit is contained in:
Yuki Okushi 2021-03-10 08:01:24 +09:00 committed by GitHub
parent 3a5d45f68c 4bceb294f4
commit 25fd50412e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 278 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
compiler/rustc_codegen_cranelift/src/base.rs
View File

@ -832,6 +832,27 @@ fn codegen_stmt<'tcx>(
}
}
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
src,
dst,
count,
}) => {
let dst = codegen_operand(fx, dst);
let pointee = dst
.layout()
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
.expect("Expected pointer");
let dst = dst.load_scalar(fx);
let src = codegen_operand(fx, src).load_scalar(fx);
let count = codegen_operand(fx, count).load_scalar(fx);
let elem_size: u64 = pointee.size.bytes();
let bytes = if elem_size != 1 {
fx.bcx.ins().imul_imm(count, elem_size as i64)
} else {
count
};
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
}
}
}

1
compiler/rustc_codegen_ssa/src/lib.rs
View File

@ -9,6 +9,7 @@
#![feature(or_patterns)]
#![feature(associated_type_bounds)]
#![recursion_limit = "256"]
#![feature(box_syntax)]
//! This crate contains codegen code that is used by all codegen backends (LLVM and others).
//! The backend-agnostic functions of this crate use functions defined in various traits that

118
compiler/rustc_codegen_ssa/src/mir/block.rs
View File

@ -641,67 +641,73 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return;
}
if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
let intrinsic = intrinsic.unwrap();
let dest = match ret_dest {
_ if fn_abi.ret.is_indirect() => llargs[0],
ReturnDest::Nothing => {
bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
}
ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
ReturnDest::DirectOperand(_) => {
bug!("Cannot use direct operand with an intrinsic call")
}
};
let args: Vec<_> = args
.iter()
.enumerate()
.map(|(i, arg)| {
// The indices passed to simd_shuffle* in the
// third argument must be constant. This is
// checked by const-qualification, which also
// promotes any complex rvalues to constants.
if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
if let mir::Operand::Constant(constant) = arg {
let c = self.eval_mir_constant(constant);
let (llval, ty) = self.simd_shuffle_indices(
&bx,
constant.span,
constant.literal.ty,
c,
);
return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty) };
} else {
span_bug!(span, "shuffle indices must be constant");
}
match intrinsic {
None | Some(sym::drop_in_place) => {}
Some(sym::copy_nonoverlapping) => unreachable!(),
Some(intrinsic) => {
let dest = match ret_dest {
_ if fn_abi.ret.is_indirect() => llargs[0],
ReturnDest::Nothing => {
bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
}
ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
ReturnDest::DirectOperand(_) => {
bug!("Cannot use direct operand with an intrinsic call")
}
};
self.codegen_operand(&mut bx, arg)
})
.collect();
let args: Vec<_> = args
.iter()
.enumerate()
.map(|(i, arg)| {
// The indices passed to simd_shuffle* in the
// third argument must be constant. This is
// checked by const-qualification, which also
// promotes any complex rvalues to constants.
if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
if let mir::Operand::Constant(constant) = arg {
let c = self.eval_mir_constant(constant);
let (llval, ty) = self.simd_shuffle_indices(
&bx,
constant.span,
constant.literal.ty,
c,
);
return OperandRef {
val: Immediate(llval),
layout: bx.layout_of(ty),
};
} else {
span_bug!(span, "shuffle indices must be constant");
}
}
Self::codegen_intrinsic_call(
&mut bx,
*instance.as_ref().unwrap(),
&fn_abi,
&args,
dest,
span,
);
self.codegen_operand(&mut bx, arg)
})
.collect();
if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
Self::codegen_intrinsic_call(
&mut bx,
*instance.as_ref().unwrap(),
&fn_abi,
&args,
dest,
span,
);
if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
}
if let Some((_, target)) = *destination {
helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
helper.funclet_br(self, &mut bx, target);
} else {
bx.unreachable();
}
return;
}
if let Some((_, target)) = *destination {
helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
helper.funclet_br(self, &mut bx, target);
} else {
bx.unreachable();
}
return;
}
// Split the rust-call tupled arguments off.

13
compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
View File

@ -125,19 +125,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let offset = args[1].immediate();
bx.gep(ptr, &[offset])
}
sym::copy_nonoverlapping => {
copy_intrinsic(
bx,
false,
false,
substs.type_at(0),
args[1].immediate(),
args[0].immediate(),
args[2].immediate(),
);
return;
}
sym::copy => {
copy_intrinsic(
bx,

20
compiler/rustc_codegen_ssa/src/mir/statement.rs
View File

@ -115,6 +115,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
self.codegen_coverage(&mut bx, coverage.clone());
bx
}
mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
ref src,
ref dst,
ref count,
}) => {
let dst_val = self.codegen_operand(&mut bx, dst);
let src_val = self.codegen_operand(&mut bx, src);
let count = self.codegen_operand(&mut bx, count).immediate();
let pointee_layout = dst_val
.layout
.pointee_info_at(&mut bx, rustc_target::abi::Size::ZERO)
.expect("Expected pointer");
let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes()));
let align = pointee_layout.align;
let dst = dst_val.immediate();
let src = src_val.immediate();
bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
bx
}
mir::StatementKind::FakeRead(..)
| mir::StatementKind::Retag { .. }
| mir::StatementKind::AscribeUserType(..)

20
compiler/rustc_middle/src/mir/mod.rs
View File

@ -1541,6 +1541,11 @@ pub enum StatementKind<'tcx> {
/// counter varible at runtime, each time the code region is executed.
Coverage(Box<Coverage>),
/// Denotes a call to the intrinsic function copy_overlapping, where `src_dst` denotes the
/// memory being read from and written to(one field to save memory), and size
/// indicates how many bytes are being copied over.
CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
/// No-op. Useful for deleting instructions without affecting statement indices.
Nop,
}
@ -1659,6 +1664,13 @@ impl Debug for Statement<'_> {
write!(fmt, "Coverage::{:?}", coverage.kind)
}
}
CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
ref src,
ref dst,
ref count,
}) => {
write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
}
Nop => write!(fmt, "nop"),
}
}
@ -1670,6 +1682,14 @@ pub struct Coverage {
pub code_region: Option<CodeRegion>,
}
#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable)]
pub struct CopyNonOverlapping<'tcx> {
pub src: Operand<'tcx>,
pub dst: Operand<'tcx>,
/// Number of elements to copy from src to dest, not bytes.
pub count: Operand<'tcx>,
}
///////////////////////////////////////////////////////////////////////////
// Places

9
compiler/rustc_middle/src/mir/visit.rs
View File

@ -436,6 +436,15 @@ macro_rules! make_mir_visitor {
location
)
}
StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
ref $($mutability)? src,
ref $($mutability)? dst,
ref $($mutability)? count,
}) => {
self.visit_operand(src, location);
self.visit_operand(dst, location);
self.visit_operand(count, location)
}
StatementKind::Nop => {}
}
}

9
compiler/rustc_mir/src/borrow_check/invalidation.rs
View File

@ -92,6 +92,15 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
self.consume_operand(location, input);
}
}
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
ref src,
ref dst,
ref count,
}) => {
self.consume_operand(location, src);
self.consume_operand(location, dst);
self.consume_operand(location, count);
}
StatementKind::Nop
| StatementKind::Coverage(..)
| StatementKind::AscribeUserType(..)

9
compiler/rustc_mir/src/borrow_check/mod.rs
View File

@ -626,6 +626,15 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
self.consume_operand(location, (input, span), flow_state);
}
}
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
..
}) => {
span_bug!(
span,
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
)
}
StatementKind::Nop
| StatementKind::Coverage(..)
| StatementKind::AscribeUserType(..)

6
compiler/rustc_mir/src/borrow_check/type_check/mod.rs
View File

@ -1520,6 +1520,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
);
}
}
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
..
}) => span_bug!(
stmt.source_info.span,
"Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
),
StatementKind::FakeRead(..)
| StatementKind::StorageLive(..)
| StatementKind::StorageDead(..)

1
compiler/rustc_mir/src/dataflow/impls/borrows.rs
View File

@ -305,6 +305,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
| mir::StatementKind::Retag { .. }
| mir::StatementKind::AscribeUserType(..)
| mir::StatementKind::Coverage(..)
| mir::StatementKind::CopyNonOverlapping(..)
| mir::StatementKind::Nop => {}
}
}

1
compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs
View File

@ -149,6 +149,7 @@ impl<'mir, 'tcx> dataflow::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir,
| StatementKind::FakeRead(..)
| StatementKind::Nop
| StatementKind::Retag(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::StorageLive(..) => {}
}
}

1
compiler/rustc_mir/src/dataflow/move_paths/builder.rs
View File

@ -318,6 +318,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
StatementKind::Retag { .. }
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Nop => {}
}
}

24
compiler/rustc_mir/src/interpret/intrinsics.rs
View File

@ -323,28 +323,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let result = Scalar::from_uint(truncated_bits, layout.size);
self.write_scalar(result, dest)?;
}
sym::copy | sym::copy_nonoverlapping => {
let elem_ty = instance.substs.type_at(0);
let elem_layout = self.layout_of(elem_ty)?;
let count = self.read_scalar(&args[2])?.to_machine_usize(self)?;
let elem_align = elem_layout.align.abi;
let size = elem_layout.size.checked_mul(count, self).ok_or_else(|| {
err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
})?;
let src = self.read_scalar(&args[0])?.check_init()?;
let src = self.memory.check_ptr_access(src, size, elem_align)?;
let dest = self.read_scalar(&args[1])?.check_init()?;
let dest = self.memory.check_ptr_access(dest, size, elem_align)?;
if let (Some(src), Some(dest)) = (src, dest) {
self.memory.copy(
src,
dest,
size,
intrinsic_name == sym::copy_nonoverlapping,
)?;
}
sym::copy => {
self.copy(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
}
sym::offset => {
let ptr = self.read_scalar(&args[0])?.check_init()?;

36
compiler/rustc_mir/src/interpret/step.rs
View File

@ -2,6 +2,7 @@
//!
//! The main entry point is the `step` method.
use crate::interpret::OpTy;
use rustc_middle::mir;
use rustc_middle::mir::interpret::{InterpResult, Scalar};
use rustc_target::abi::LayoutOf;
@ -113,6 +114,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
M::retag(self, *kind, &dest)?;
}
// Call CopyNonOverlapping
CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { dst, src, count }) => {
let count = self.eval_operand(count, None)?;
let src = self.eval_operand(src, None)?;
let dst = self.eval_operand(dst, None)?;
self.copy(&src, &dst, &count, /* nonoverlapping */ true)?;
}
// Statements we do not track.
AscribeUserType(..) => {}
@ -140,6 +150,32 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Ok(())
}
pub(crate) fn copy(
&mut self,
src: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
dst: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
count: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>,
nonoverlapping: bool,
) -> InterpResult<'tcx> {
let count = self.read_scalar(&count)?.to_machine_usize(self)?;
let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?;
let (size, align) = (layout.size, layout.align.abi);
let src =
self.memory.check_ptr_access(self.read_scalar(&src)?.check_init()?, size, align)?;
let dst =
self.memory.check_ptr_access(self.read_scalar(&dst)?.check_init()?, size, align)?;
let size = size.checked_mul(count, self).ok_or_else(|| {
err_ub_format!("overflow computing total size of `copy_nonoverlapping`")
})?;
if let (Some(src), Some(dst)) = (src, dst) {
self.memory.copy(src, dst, size, nonoverlapping)?;
}
Ok(())
}
/// Evaluate an assignment statement.
///
/// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue

1
compiler/rustc_mir/src/transform/check_consts/validation.rs
View File

@ -808,6 +808,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
| StatementKind::Retag { .. }
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Nop => {}
}
}

1
compiler/rustc_mir/src/transform/check_unsafety.rs
View File

@ -123,6 +123,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> {
UnsafetyViolationKind::General,
UnsafetyViolationDetails::UseOfInlineAssembly,
),
StatementKind::CopyNonOverlapping(..) => unreachable!(),
}
self.super_statement(statement, location);
}

1
compiler/rustc_mir/src/transform/coverage/spans.rs
View File

@ -687,6 +687,7 @@ pub(super) fn filtered_statement_span(
// Retain spans from all other statements
StatementKind::FakeRead(_, _) // Not including `ForGuardBinding`
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Assign(_)
| StatementKind::SetDiscriminant { .. }
| StatementKind::LlvmInlineAsm(_)

1
compiler/rustc_mir/src/transform/dest_prop.rs
View File

@ -587,6 +587,7 @@ impl Conflicts<'a> {
| StatementKind::FakeRead(..)
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Nop => {}
}
}

1
compiler/rustc_mir/src/transform/generator.rs
View File

@ -1454,6 +1454,7 @@ impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
| StatementKind::Retag(..)
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Nop => {}
}
}

21
compiler/rustc_mir/src/transform/lower_intrinsics.rs
View File

@ -40,6 +40,27 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
terminator.kind = TerminatorKind::Goto { target };
}
}
sym::copy_nonoverlapping => {
let target = destination.unwrap().1;
let mut args = args.drain(..);
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::CopyNonOverlapping(
box rustc_middle::mir::CopyNonOverlapping {
src: args.next().unwrap(),
dst: args.next().unwrap(),
count: args.next().unwrap(),
},
),
});
assert_eq!(
args.next(),
None,
"Extra argument for copy_non_overlapping intrinsic"
);
drop(args);
terminator.kind = TerminatorKind::Goto { target };
}
sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
if let Some((destination, target)) = *destination {
let lhs;

1
compiler/rustc_mir/src/transform/remove_noop_landing_pads.rs
View File

@ -55,6 +55,7 @@ impl RemoveNoopLandingPads {
StatementKind::Assign { .. }
| StatementKind::SetDiscriminant { .. }
| StatementKind::LlvmInlineAsm { .. }
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Retag { .. } => {
return false;
}

1
compiler/rustc_mir/src/transform/simplify.rs
View File

@ -428,6 +428,7 @@ impl Visitor<'_> for UsedLocals {
fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
match statement.kind {
StatementKind::LlvmInlineAsm(..)
| StatementKind::CopyNonOverlapping(..)
| StatementKind::Retag(..)
| StatementKind::Coverage(..)
| StatementKind::FakeRead(..)

44
compiler/rustc_mir/src/transform/validate.rs
View File

@ -294,7 +294,49 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
);
}
}
_ => {}
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
ref src,
ref dst,
ref count,
}) => {
let src_ty = src.ty(&self.body.local_decls, self.tcx);
let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
src_deref.ty
} else {
self.fail(
location,
format!("Expected src to be ptr in copy_nonoverlapping, got: {}", src_ty),
);
return;
};
let dst_ty = dst.ty(&self.body.local_decls, self.tcx);
let op_dst_ty = if let Some(dst_deref) = dst_ty.builtin_deref(true) {
dst_deref.ty
} else {
self.fail(
location,
format!("Expected dst to be ptr in copy_nonoverlapping, got: {}", dst_ty),
);
return;
};
// since CopyNonOverlapping is parametrized by 1 type,
// we only need to check that they are equal and not keep an extra parameter.
if op_src_ty != op_dst_ty {
self.fail(location, format!("bad arg ({:?} != {:?})", op_src_ty, op_dst_ty));
}
let op_cnt_ty = count.ty(&self.body.local_decls, self.tcx);
if op_cnt_ty != self.tcx.types.usize {
self.fail(location, format!("bad arg ({:?} != usize)", op_cnt_ty))
}
}
StatementKind::SetDiscriminant { .. }
| StatementKind::StorageLive(..)
| StatementKind::StorageDead(..)
| StatementKind::LlvmInlineAsm(..)
| StatementKind::Retag(_, _)
| StatementKind::Coverage(_)
| StatementKind::Nop => {}
}
self.super_statement(statement, location);

1
compiler/rustc_mir/src/util/spanview.rs
View File

@ -245,6 +245,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
Retag(..) => "Retag",
AscribeUserType(..) => "AscribeUserType",
Coverage(..) => "Coverage",
CopyNonOverlapping(..) => "CopyNonOverlapping",
Nop => "Nop",
}
}

9
src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
View File

@ -210,7 +210,7 @@ fn check_statement(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: DefId, statemen
StatementKind::Assign(box (place, rval)) => {
check_place(tcx, *place, span, body)?;
check_rvalue(tcx, body, def_id, rval, span)
},
}
StatementKind::FakeRead(_, place) |
// just an assignment
@ -218,6 +218,13 @@ fn check_statement(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: DefId, statemen
StatementKind::LlvmInlineAsm { .. } => Err((span, "cannot use inline assembly in const fn".into())),
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping{
dst, src, count,
}) => {
check_operand(tcx, dst, span, body)?;
check_operand(tcx, src, span, body)?;
check_operand(tcx, count, span, body)
}
// These are all NOPs
StatementKind::StorageLive(_)
| StatementKind::StorageDead(_)